Posts on Rahat Zaman

A Declarative Layer Visualization Grammar for Deep Learning Models

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 01 Sep 2025 22:22:50 +0600

A Declarative Layer Visualization Grammar for Deep Learning Models

This post outlines a compact, Layer Visualization Grammar (LVG) for building scalable, reproducible visualizations across different models like CNNs, LSTM, Transformers, and Diffusion models. The grammar supports both generic tensor views and targeted model-specific views, and it uses a global state manager to coordinate interactions across panels and sessions.

Core Spec Shape

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46


interface LVGSpec {
 $schema: "https://lvg.dev/v1";
 id?: string;
 title?: string;
 description?: string;

 target: {
 model_id: string; // runtime handle
 layer: string; // e.g., "blocks.5.attn"
 pass?: "train" | "eval" | "inference";
 scope?: "forward" | "backward";
 batch?: number | "active";
 };

 inputs?: InputBinding[];
 parameters?: Param[];
 signals?: Signal[];
 state?: StateBindings; // global state read/write

 views: (GenericView | TargetedView | CompositeView)[];
 layout?: { type?: "grid" | "hstack" | "vstack" | "tabs"; columns?: number; gap?: number; responsive?: "none" | "wrap" };
}

type InputBinding = {
 as: string;
 from: TensorRef | MetaRef;
 slice?: SliceSpec;
 normalize?: "none" | "layer" | "channel" | "token" | "minmax" | "zscore";
};

type TensorRef = { type: "tensor"; source: "layer_output" | "layer_input" | "param" | "upstream" | "downstream" | "cache"; name?: string };
type MetaRef = { type: "meta"; key: "shape" | "param_count" | "receptive_field" | "token_strings" | "class_labels" | "timesteps" };
type SliceSpec = { axes?: Array<number | "last">; index?: Array<number | { start?: number; end?: number; step?: number } | "all"> };

type Param = {
 name: string; value: number | string | boolean | number[] | string[];
 ui?: { type: "slider" | "select" | "checkbox" | "text" | "range"; min?: number; max?: number; step?: number; options?: Array<{label:string; value:any}> };
};

type Signal = { name: string; value?: any; on?: Array<{ event: UIEvent; update: Expr }> };
type UIEvent = { type: "pointerhover" | "click" | "brush" | "keydown"; view?: string; key?: string };

type StateBindings = {
 read?: Array<{ key: string; as: string; default?: any }>;
 write?: Array<{ key: string; from: Expr | string; on: UIEvent | "immediate" }>;
};

Views, Transforms, and Encodings

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78


interface GenericView {
 type: "generic";
 id: string;
 title?: string;
 data: Dataflow;
 encoding: Encoding;
 interactions?: Interaction[];
 actions?: Action[];
}

interface TargetedView {
 type: "targeted";
 id: string;
 title?: string;
 preset:
 | "transformer_attention_matrix"
 | "transformer_head_grid"
 | "cnn_channel_heatmap"
 | "cnn_featuremap_gallery"
 | "diffusion_step_preview"
 | "diffusion_noise_to_image"
 | "rnn_gate_timelines"
 | "token_saliency_grid";
 inputs?: Record<string, string>;
 options?: Record<string, any>;
 interactions?: Interaction[];
 actions?: Action[];
}

interface CompositeView {
 type: "composite";
 id: string;
 title?: string;
 layout: LVGSpec["layout"];
 children: (GenericView | TargetedView)[];
}

type Dataflow = {
 sources: Array<{ name: string; from: string; kind: "tensor" | "meta" | "state" | "literal"; value?: any }>;
 transforms: Transform[];
 output: string;
};

type Transform =
 | { op: "reshape"; from: string; to: "table" | "matrix"; axis_names?: string[] }
 | { op: "slice"; from: string; spec: SliceSpec; as: string }
 | { op: "reduce"; from: string; dims?: number[]; fn: "mean" | "max" | "min" | "sum" | "std" | "var" | "norm"; keepdims?: boolean; as: string }
 | { op: "normalize"; from: string; mode: "minmax" | "zscore" | "l2"; along?: "row" | "col" | "global"; as: string }
 | { op: "project"; from: string; method: "pca" | "umap" | "tsne"; k: number; seed?: number; metric?: "cosine" | "euclidean"; as: string }
 | { op: "similarity"; from: string; against?: string; metric: "cosine" | "dot" | "jaccard"; topk?: number; threshold?: number; as: string }
 | { op: "compare"; a: string; b: string; mode: "diff" | "ratio" | "corr"; as: string }
 | { op: "rank"; from: string; by: string; order?: "asc" | "desc"; groupby?: string[]; as: string }
 | { op: "quantiles"; from: string; q: number[]; as: string }
 | { op: "aggregate"; from: string; by: string[]; ops: Array<{ fn: "mean"|"max"|"count"|"sum"|"std"; as: string }> }
 | { op: "threshold"; from: string; by: string; op: ">" | "<" | ">=" | "<="; value: number; as: string }
 | { op: "topk"; from: string; by: string; k: number; as: string }
 | { op: "join"; left: string; right: string; on: string[]; how: "inner" | "left" | "right" | "outer"; as: string }
 | { op: "expr"; from: string; expr: Expr; as: string };

type Encoding = {
 mark: "scatter" | "heatmap" | "image" | "line" | "bar" | "table";
 data: string;
 x?: Channel; y?: Channel; color?: Channel; size?: Channel; shape?: Channel; text?: Channel;
 axes?: { x?: Axis; y?: Axis };
 legend?: boolean | { position?: "right" | "bottom" };
 tooltip?: string[];
};

type Channel = { field: string; type: "quant" | "ordinal" | "nominal" | "index" };
type Axis = { label?: string; grid?: boolean; tickCount?: number };

type Interaction =
 | { type: "brush"; target: "x" | "y" | "xy"; signal: string }
 | { type: "select"; on: "click" | "hover"; fields: string[]; signal: string }
 | { type: "link"; with: string; using: string[] };

type Action =
 | { id: string; label: string; kind: "export_csv" | "export_png" | "write_state" | "mutate_model" | "compare_with_state"; payload?: any; from?: string; toStateKey?: string };

Targeted Presets: Contracts

transformer_attention_matrix
Inputs: attn [batch, heads, tokens, tokens], optional token_strings.
Options: head, softmax_temp, normalize_along (row|col).
transformer_head_grid
Inputs: Q, K, V.
Options: head selector; shows norms and head-wise similarities.
cnn_channel_heatmap
Inputs: acts [batch, channels, H, W].
Options: summary (l2|mean|max), rank_by (class_separation|variance).
cnn_featuremap_gallery
Inputs: acts.
Options: topk, normalize, tiling mode.
diffusion_step_preview
Inputs: x_t, eps_pred, timesteps, scheduler params.
Options: scheduler, eta, t, decode.
diffusion_noise_to_image
Inputs: same as above; runs a short denoise loop from t to t-k.
rnn_gate_timelines
Inputs: i,f,o,g over time.
Options: sequence range, aggregation.
token_saliency_grid
Inputs: saliency [tokens] or [layers,tokens], optional token_strings.
Options: normalization, threshold.

Example: Transformer Attention Debugger

The following spec composes generic and targeted views for a Transformer self-attention layer. It:

Projects token embeddings (from V) via DR and allows brushing.
Shows a head-selectable attention matrix targeted view.
Compares the current head’s attention submatrix to a pinned baseline stored in global state.

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129


{
 "$schema": "https://lvg.dev/v1",
 "id": "transformer-attn-debugger",
 "title": "Transformer Attention Debugger",
 "target": {
 "model_id": "active_model",
 "layer": "encoder.blocks.5.self_attn",
 "pass": "inference",
 "scope": "forward",
 "batch": "active"
 },
 "inputs": [
 { "as": "Q", "from": { "type": "tensor", "source": "layer_output", "name": "Q" } },
 { "as": "K", "from": { "type": "tensor", "source": "layer_output", "name": "K" } },
 { "as": "V", "from": { "type": "tensor", "source": "layer_output", "name": "V" } },
 { "as": "ATTN", "from": { "type": "tensor", "source": "layer_output", "name": "attn" } },
 { "as": "TOKENS", "from": { "type": "meta", "key": "token_strings" } }
 ],
 "parameters": [
 { "name": "head", "value": 0, "ui": { "type": "slider", "min": 0, "max": 11, "step": 1 } },
 {
 "name": "dr_method",
 "value": "pca",
 "ui": { "type": "select", "options": [ { "label": "PCA", "value": "pca" }, { "label": "UMAP", "value": "umap" } ] }
 }
 ],
 "signals": [
 { "name": "hover_token", "value": null },
 { "name": "selected_tokens", "value": [] }
 ],
 "state": {
 "read": [
 { "key": "debug.selected_tokens", "as": "selected_tokens", "default": [] },
 { "key": "debug.selected_head", "as": "head", "default": 0 }
 ],
 "write": [
 { "key": "debug.selected_tokens", "from": "selected_tokens", "on": { "type": "brush", "view": "token-embed" } },
 { "key": "debug.selected_head", "from": "head", "on": "immediate" }
 ]
 },
 "layout": { "type": "grid", "columns": 2, "gap": 12, "responsive": "wrap" },
 "views": [
 {
 "type": "generic",
 "id": "token-embed",
 "title": "Token Embedding (V) — DR Scatter",
 "data": {
 "sources": [
 { "name": "V_src", "from": "V", "kind": "tensor" },
 { "name": "TOK", "from": "TOKENS", "kind": "meta" }
 ],
 "transforms": [
 { "op": "reshape", "from": "V_src", "to": "table", "axis_names": ["batch","tokens","heads","dim"] },
 { "op": "reduce", "from": "V_src", "dims": [2], "fn": "mean", "keepdims": false, "as": "V_tokens" },
 { "op": "project", "from": "V_tokens", "method": { "$param": "dr_method" }, "k": 2, "as": "V_2d" },
 { "op": "expr", "from": "V_2d", "expr": "add_field(index(),'tok_idx')", "as": "V_xy" },
 { "op": "join", "left": "V_xy", "right": "TOK", "on": ["tok_idx"], "how": "left", "as": "V_labeled" }
 ],
 "output": "V_labeled"
 },
 "encoding": {
 "mark": "scatter",
 "data": "V_labeled",
 "x": { "field": "x0", "type": "quant" },
 "y": { "field": "x1", "type": "quant" },
 "color": { "field": "tok_idx", "type": "index" },
 "size": { "field": "variance", "type": "quant" },
 "tooltip": ["tok_idx", "token_string"]
 },
 "interactions": [
 { "type": "brush", "target": "xy", "signal": "selected_tokens" },
 { "type": "select", "on": "hover", "fields": ["tok_idx"], "signal": "hover_token" }
 ],
 "actions": [
 { "id": "export-embed", "label": "Export CSV", "kind": "export_csv", "from": "V_labeled" }
 ]
 },
 {
 "type": "targeted",
 "id": "attn-matrix",
 "title": "Attention Matrix — Head & Token Subset",
 "preset": "transformer_attention_matrix",
 "inputs": { "attn": "ATTN", "token_strings": "TOKENS" },
 "options": { "head": { "$param": "head" }, "normalize_along": "row" },
 "interactions": [
 { "type": "link", "with": "token-embed", "using": ["tok_idx"] }
 ],
 "actions": [
 { "id": "pin-head", "label": "Set as Compare A", "kind": "write_state", "toStateKey": "debug.compare.attnA", "from": "head" },
 { "id": "pin-selection", "label": "Set Tokens as Compare A", "kind": "write_state", "toStateKey": "debug.compare.tokensA", "from": "selected_tokens" }
 ]
 },
 {
 "type": "generic",
 "id": "attn-compare",
 "title": "Compare Current Head vs Pinned",
 "data": {
 "sources": [
 { "name": "ATTN_src", "from": "ATTN", "kind": "tensor" },
 { "name": "head_curr", "from": "head", "kind": "state" },
 { "name": "head_A", "from": "debug.compare.attnA", "kind": "state" },
 { "name": "sel_tokens", "from": "selected_tokens", "kind": "state" },
 { "name": "tokensA", "from": "debug.compare.tokensA", "kind": "state" }
 ],
 "transforms": [
 { "op": "slice", "from": "ATTN_src", "spec": { "axes": [0,1,2,3], "index": ["all", { "$state": "head_curr" }, "all", "all"] }, "as": "A_curr" },
 { "op": "slice", "from": "ATTN_src", "spec": { "axes": [0,1,2,3], "index": ["all", { "$state": "head_A" }, "all", "all"] }, "as": "A_pin" },
 { "op": "reduce", "from": "A_curr", "dims": [0], "fn": "mean", "as": "A_curr2D" },
 { "op": "reduce", "from": "A_pin", "dims": [0], "fn": "mean", "as": "A_pin2D" },
 { "op": "slice", "from": "A_curr2D", "spec": { "axes": [0,1], "index": [ { "$state": "sel_tokens" }, { "$state": "sel_tokens" } ] }, "as": "A_curr_sub" },
 { "op": "slice", "from": "A_pin2D", "spec": { "axes": [0,1], "index": [ { "$state": "tokensA" }, { "$state": "tokensA" } ] }, "as": "A_pin_sub" },
 { "op": "compare", "a": "A_curr_sub", "b": "A_pin_sub", "mode": "diff", "as": "A_diff" }
 ],
 "output": "A_diff"
 },
 "encoding": {
 "mark": "heatmap",
 "data": "A_diff",
 "x": { "field": "col", "type": "index" },
 "y": { "field": "row", "type": "index" },
 "color": { "field": "value", "type": "quant" },
 "tooltip": ["row","col","value"]
 },
 "actions": [
 { "id": "export-diff", "label": "Export Diff CSV", "kind": "export_csv", "from": "A_diff" }
 ]
 }
 ]
}

CNN Example: Channel Heatmap + DR Scatter

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48


{
 "$schema": "https://lvg.dev/v1",
 "id": "cnn-channels",
 "title": "CNN Channel Heatmap + DR",
 "target": { "model_id": "active_model", "layer": "features.17", "pass": "inference", "scope": "forward", "batch": "active" },
 "inputs": [
 { "as": "ACTS", "from": { "type": "tensor", "source": "layer_output", "name": "activations" } },
 { "as": "LABELS", "from": { "type": "meta", "key": "class_labels" } }
 ],
 "parameters": [
 { "name": "summary", "value": "l2", "ui": { "type": "select", "options": [ { "label": "L2", "value": "l2" }, { "label": "Mean", "value": "mean" }, { "label": "Max", "value": "max" } ] } },
 { "name": "topk", "value": 64, "ui": { "type": "slider", "min": 8, "max": 256, "step": 8 } }
 ],
 "layout": { "type": "grid", "columns": 2, "gap": 12 },
 "views": [
 {
 "type": "targeted",
 "id": "chan-heat",
 "title": "Channel Heatmap (summary × images)",
 "preset": "cnn_channel_heatmap",
 "inputs": { "acts": "ACTS" },
 "options": { "summary": { "$param": "summary" }, "rank_by": "variance", "topk": { "$param": "topk" } }
 },
 {
 "type": "generic",
 "id": "chan-dr",
 "title": "Channel DR (H×W reduced)",
 "data": {
 "sources": [ { "name": "A", "from": "ACTS", "kind": "tensor" } ],
 "transforms": [
 { "op": "reduce", "from": "A", "dims": [2,3], "fn": "norm", "as": "A_ch" }, // [batch, channels]
 { "op": "reduce", "from": "A_ch", "dims": [0], "fn": "mean", "as": "A_ch_mean" }, // [channels]
 { "op": "project", "from": "A_ch_mean", "method": "pca", "k": 2, "as": "A_2d" },
 { "op": "expr", "from": "A_2d", "expr": "add_field(index(),'ch')", "as": "A_xy" }
 ],
 "output": "A_xy"
 },
 "encoding": {
 "mark": "scatter",
 "data": "A_xy",
 "x": { "field": "x0", "type": "quant" },
 "y": { "field": "x1", "type": "quant" },
 "color": { "field": "ch", "type": "index" },
 "tooltip": ["ch"]
 }
 }
 ]
}

Diffusion Example: Single-Step Preview

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33


{
 "$schema": "https://lvg.dev/v1",
 "id": "diffusion-preview",
 "title": "Diffusion Step Preview (DDIM)",
 "target": { "model_id": "sd15", "layer": "unet.down_blocks.2", "pass": "inference", "scope": "forward", "batch": "active" },
 "inputs": [
 { "as": "X_T", "from": { "type": "tensor", "source": "upstream", "name": "x_t" } },
 { "as": "EPS", "from": { "type": "tensor", "source": "layer_output", "name": "eps_pred" } },
 { "as": "T", "from": { "type": "meta", "key": "timesteps" } }
 ],
 "parameters": [
 { "name": "t", "value": 350, "ui": { "type": "slider", "min": 0, "max": 999, "step": 1 } },
 { "name": "eta", "value": 0.0, "ui": { "type": "slider", "min": 0, "max": 1, "step": 0.05 } }
 ],
 "state": {
 "write": [
 { "key": "diff.t", "from": "t", "on": "immediate" }
 ]
 },
 "views": [
 {
 "type": "targeted",
 "id": "ddim-preview",
 "title": "x_{t-1} and Decoded Image",
 "preset": "diffusion_step_preview",
 "inputs": { "x_t": "X_T", "eps_pred": "EPS", "timesteps": "T" },
 "options": { "scheduler": "ddim", "eta": { "$param": "eta" }, "t": { "$param": "t" }, "decode": true },
 "actions": [
 { "id": "pin-t", "label": "Pin t", "kind": "write_state", "toStateKey": "diff.t_pinned", "from": "t" }
 ]
 }
 ]
}

Implementation Notes

Backend adapters: Resolve TensorRef to framework tensors; convert to ndarray for transforms. Cache transform outputs keyed by (model_id, layer, input_name, params_hash).
Presets: Thin wrappers over generic transforms with domain helpers (e.g., softmax temperature, head tiling, DDIM step).
Global state: JSON-serializable values with subscriptions; updates trigger recomputation of dependent specs.
Interactivity: Brushing and linking rely on shared identifiers (tok_idx, ch, t) across views.

JsPsych Experiment with Vite and TypeScript

rahatzamancse@gmail.com (Rahat Zaman) — Wed, 23 Oct 2024 10:21:45 -0600

In this guide, we’ll walk you through setting up a simple psychology experiment using JsPsych within a Vite and TypeScript project. We’ll assume you have little to no prior experience with JsPsych, Vite, or TypeScript. This step-by-step guide will explain the basics and get you up and running quickly.

What is JsPsych?

JsPsych is a library for creating behavioral experiments that run in a web browser. It is often used in psychology and cognitive science research to create experiments that participants can complete online.

Why Use Vite and TypeScript with JsPsych?

Vite helps you develop web applications quickly, refreshing the page whenever you make changes, which speeds up development.
TypeScript ensures that your code is easier to read and less prone to errors by adding extra checks.
JsPsych provides the structure and tools needed to create an experiment, such as displaying stimuli and recording participant responses.

Now, let’s set up everything step by step.

Step 1: Set Up a Vite + TypeScript Project

If you haven’t already set up a Vite + TypeScript project, follow these instructions:

1.1 Open Your Terminal

Start by opening your Terminal (or Command Prompt on Windows).

1.2 Create a New Vite Project

In the terminal, type the following command and press Enter:

1

npm create vite@latest

Answer the questions:

Project name: Enter a name (e.g., jspsych-experiment).
Framework: Choose vanilla (basic JavaScript).
Variant: Choose TypeScript.

Once the project is created, move into your new project folder:

1

cd jspsych-experiment

1.3 Install Dependencies

Run the following command to install all the required files:

1

npm install

To know more about what these steps are doing, check out the Introduction to Vite + Typescript tutorial.

Now, you’ve set up your Vite + TypeScript project!

Step 2: Install JsPsych

JsPsych is not included by default in your Vite project, so you need to install it manually.

In the terminal, run the following command to add JsPsych to your project:

1
2


npm install jspsych # to install the core library
npm install @jspsych/plugin-button-response # to install the button-response plugin

This will download JsPsych and add it to your project’s dependencies.

Step 3: Modify Your Project Files

Next, we’ll edit the project files to display a simple JsPsych experiment.

3.1 Set Up Your HTML File

First, let’s modify the index.html file, which is the main web page for your experiment.

Open index.html in your code editor (like VS Code).

Modify it to include a placeholder where the JsPsych experiment will be displayed:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14


<!DOCTYPE html>
<html lang="en">
<head>
 <meta charset="UTF-8" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 <title>JsPsych Experiment</title>
</head>
<body>
 <h1>Welcome to My JsPsych Experiment</h1>
 <div id="jspsych-experiment"></div>

 <script type="module" src="/src/main.ts"></script>
</body>
</html>

3.2 Create a Simple JsPsych Experiment

Now, let’s write the code for a basic JsPsych experiment in main.ts. This code will create a simple trial where the participant sees a message and responds by pressing a key.

Open src/main.ts in your editor.

Add the following code:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17


import { initJsPsych } from 'jspsych';
import jsPsychButtonResponse from '@jspsych/plugin-button-response';

const jsPsych: JsPsych = jsPsychInit();

// Create a simple timeline with one trial
const timeline: any[] = [];

const trial = {
 type: jsPsychButtonResponse,
 stimulus: '<p>Press any key to begin the experiment.</p>',
};

timeline.push(trial);

// Initialize the JsPsych experiment
jsPsych.run(timeline);

Breakdown of the Code:

import { initJsPsych } from 'jspsych';: This imports the JsPsych library into your project.
const jsPsych: JsPsych = initJsPsych();: This initializes JsPsych and assigns it to the jsPsych variable.
const timeline: any[] = [];: A timeline in JsPsych is a list of trials that define your experiment. We create an empty timeline and add our trials to it.
const trial = {...}: This is a basic trial that presents a message and waits for the participant to press any key.
jsPsych.run(timeline);: This starts the experiment using the timeline we created.

Step 4: Run the Experiment

With everything set up, you’re ready to run the experiment and see it in action!

Open the terminal (if it’s not already open).
Run the development server with this command:
1

npm run dev

Vite will start a local development server and give you an address like http://localhost:3000/. Open this address in your web browser.

You should see your experiment with the message “Press any key to begin the experiment.” When you press a key, the trial will complete.

Conclusion

You have successfully set up a simple JsPsych experiment using Vite and TypeScript! Here’s what we did in this tutorial:

Created a Vite + TypeScript project.
Installed JsPsych to run psychology experiments.
Wrote a simple experiment where a participant presses a key.
Ran the experiment in a browser.

Recap:

Vite makes your development faster by reloading changes automatically.
TypeScript helps catch mistakes in your code early.
JsPsych allows you to easily create browser-based behavioral experiments.

Now you can continue to build more complex experiments by adding more trials and customizing how participants interact with the experiment. Happy experimenting!

Introduction to Vite Typescript

rahatzamancse@gmail.com (Rahat Zaman) — Wed, 23 Oct 2024 09:58:43 -0600

This guide will walk you through setting up a project using Vite and TypeScript from scratch. Don’t worry if you’re unfamiliar with programming or technical terms. We’ll explain everything in simple terms.

What Is Vite?

Before diving into the steps, let’s first understand what Vite is and why it’s useful.

Vite (pronounced like “veet”) is a tool that helps you develop websites. When building a website, you usually need to set up many things—like how to handle multiple files and make your website load quickly. Vite takes care of all that for you, so you can focus on writing your website’s code.
It makes development faster, especially when your project gets larger.
Instead of loading everything at once (which can make your website slow), Vite loads only what’s needed when you start developing.

Why use Vite? Let’s look at an example:

Imagine you’re building a website using plain HTML and JavaScript (the language used to make websites interactive). As your website grows, it gets slower because you have to load all the files at once. Vite helps avoid this by loading just the files you’re working on, making it much faster to work with.

Why is Vite Better than Plain HTML and JavaScript?

Faster development: Vite refreshes only the part of the page you’re working on, while with plain HTML/JavaScript, you’d have to reload the entire page.
Organized code: You can split your code into smaller parts, which Vite helps combine neatly.
Support for modern technologies: It makes it easier to use new web development features like TypeScript.

What is TypeScript?

Now, let’s talk about TypeScript:

TypeScript is a programming language that builds on JavaScript. It adds extra features, such as “types,” which help you avoid common mistakes when writing code.
It helps make your code more predictable, reducing the chances of bugs.

You don’t need to fully understand TypeScript yet to follow this guide, but think of it as a “safer” version of JavaScript that helps you write cleaner, less error-prone code.

Why Use TypeScript?

Catch errors early: TypeScript checks your code for potential problems before it runs, so you can fix them before they break your website.
Better collaboration: If you’re working with other developers, TypeScript makes it easier for everyone to understand how the code works.

Now that we know why Vite and TypeScript are useful, let’s get started with setting them up.

Step 1: Setting Up Your Development Environment

Before we can use Vite and TypeScript, you’ll need to install a few basic tools:

1.1 Install Node.js

Node.js is a tool that lets you run JavaScript (and TypeScript) outside of a web browser. Vite uses Node.js to manage your website’s files during development.
Download Node.js from its official website.
- Choose the LTS version (it stands for Long Term Support, meaning it is more stable).
- Once downloaded, install it by following the on-screen instructions.

1.2 Install a Code Editor

You’ll need a place to write your code. The most popular tool for this is Visual Studio Code (VS Code):

Download Visual Studio Code from its official site and install it.
This editor comes with useful features like error checking and highlighting code, which makes it beginner-friendly.

Step 2: Starting a Vite + TypeScript Project

With Node.js installed, you can create your Vite + TypeScript project.

2.1 Open Your Terminal

On your computer, open the Terminal or Command Prompt. If you’re using Windows, search for “Command Prompt” in the Start menu.

2.2 Create a New Vite Project

Now, let’s create your first Vite project:

In the terminal, type the following command and press Enter:
1

npm create vite@latest
- This command tells Node.js to create a new project using Vite.
You’ll be asked a few questions:
- Project name: Enter a name for your project (e.g., my-vite-project).
- Framework: Choose vanilla (this means basic JavaScript).
- Variant: Choose TypeScript (this sets up TypeScript automatically for you).
Once you’ve answered the questions, the terminal will set up your project. Next, move into the project folder by typing:
1

cd my-vite-project

2.3 Install Dependencies

Next, you’ll need to install some files that Vite needs to run your project. These are called dependencies.

Run this command in your terminal:
1

npm install

This will download all the necessary files.

Step 3: Running Your Project

You’re now ready to start your Vite project and see it in action!

In the terminal, type the following command:
1

npm run dev
Vite will start a local server, and it will tell you an address to visit in your web browser, something like:
```
Local: http://localhost:3000/
```
- Open your browser and go to this address. You should see a page that says Vite + TypeScript. Congratulations! You now have a basic Vite project running with TypeScript.

Step 4: Understanding Your Project’s Structure

Let’s break down what Vite has created for you:

index.html: This is the main page of your website. It tells the browser what content to display.
main.ts: This is the main TypeScript file. It is where you will write your code to make the page interactive.
style.css: This file contains the styles for your website. It controls how things look, such as colors and layouts.

You don’t need to worry too much about these files yet. Over time, as you add more features, you’ll add more files.

Step 5: Making Changes to Your Project

Now, let’s modify your project to see how easy it is to work with Vite and TypeScript.

5.1 Changing Text on the Page

Open index.html in your code editor.
Find the line that says:
1

<h1>Vite + TypeScript</h1>
Change it to:
1

<h1>Hello, World!</h1>
Save the file.

When you go back to your browser (the one you opened in Step 3), you’ll see the text has changed without refreshing the page! This is one of Vite’s cool features—it automatically updates the page when you save changes.

5.2 Adding Some Interactivity

Open main.ts in your editor.

Add the following code at the end of the file:

1
2
3
4
5
6
7


const button = document.createElement('button');
button.textContent = 'Click Me';
document.body.appendChild(button);

button.addEventListener('click', () => {
 alert('You clicked the button!');
});

Save the file.

When you look at your browser again, you’ll see a button has appeared. If you click it, an alert will pop up! This is an example of how you can make your page interactive using TypeScript.

Conclusion

You’ve now set up a project using Vite and TypeScript, made some changes, and added a button to your webpage. By using Vite, you can develop websites quickly, and with TypeScript, you can write code that’s safer and easier to manage.

Recap:

Vite helps you develop faster by loading only what you need and refreshing changes automatically.
TypeScript helps catch potential errors before they happen, making your code more reliable.

You can now continue building your project by exploring more TypeScript features or adding more pages to your website.

Happy coding!

BTRFS + SWAP + HIBERNATE

rahatzamancse@gmail.com (Rahat Zaman) — Wed, 02 Oct 2024 22:22:50 +0600

How to enable hibernation in EndeavourOS+BTRFS+swapfile

First, we need to make sure that the swap file is at least of the size of the ram.

1

free -h

1
2
3


 total used free shared buff/cache available
Mem: 15Gi 3.3Gi 10Gi 1.1Gi 2.9Gi 11Gi
Swap: 16Gi 0B 16Gi

Ok, so it is larger than the memory. If it is not, then we need to recreate the swap file with a larger size. To do that, first stop and remove the existing swapfile (is this case, the default location of swapfile is in /swap/swapfile).

1
2


sudo swapoff /swap/swapfile # stops the swap
sudo rm -f /swap/swapfile # remove the swapfile

Then we create a new swapfile. Usually it can be done with the dd command, but btrfs has a special command already for this.

1
2


sudo btrfs filesystem mkswapfile /swap/swapfile -s 17G
sudo swapon /swap/swapfile

Now we should be a swapfile that is larger than our memory.

Next, we need to enable the resume hook for our initramfs generator. The most common generator is mkinitcpio. In that case, we can add resume in /etc/mkinitcpio.conf like below:

1

HOOKS=(base udev autodetect modconf block kbd_backlight encrypt filesystems keyboard fsck resume)

But endeavourOS has switched to Dracut. In this case, we edit the file /etc/ostree-mkinitcpio.conf and add the resume flag like below:

1

HOOKS="base udev autodetect modconf block kbd_backlight encrypt filesystems keyboard fsck resume"

Then we rebuild the initramfs with sudo dracut-rebuild.

Next we grab two important variables about the swapfile in btrfs filesystem. The first one is the UUID of the swapfile:

1

sudo findmnt -no UUID -T /swap/swapfile # <the-uuid> = a44a89ea-0560-4bff-bbf1-1855ef63e6e6

Then we get the offset of the swapfile. For non-btrfs partitions, it can be obtained with findmnt as well, but for btrfs, it does not work. For btrfs, we need to obtain the offset by:

1

sudo btrfs inspect-internal map-swapfile -r /swap/swapfile # <offset> = 373845

After getting these two information, we need to add these to /etc/default/grub in the GRUB_CMDLINE_LINUX_DEFAULT variable like below:

1
2
3
4
5
6
7


# GRUB boot loader configuration

GRUB_DEFAULT='0'
GRUB_TIMEOUT='5'
GRUB_DISTRIBUTOR='EndeavourOS'
GRUB_CMDLINE_LINUX_DEFAULT='nowatchdog nvme_load=YES loglevel=3 resume_offset=373845 resume=UUID=a44a89ea-0560-4bff-bbf1-1855ef63e6e6'
GRUB_CMDLINE_LINUX=""

Then create the grub configs by running sudo grub-mkconfig -o /boot/grub/grub.cfg.

Finally reboot the system. Now test if hibernation works by running sudo systemctl hibernate. This will turn your computer off. Then upon starting, this will boot straight to the last session you were.

Frailty Visualization

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 21 Apr 2022 13:22:57 -0700

This is very WIP project under active development. For which, I have not included a lot of details and just kept everything as draft.

The visualization website is deployed at: http://river.cs.arizona.edu:1600/blob-viz (You need to be connected to University of Arizona Wifi or use the VPN provided by UofA).

The GitHub public repository is here: github.com/enoriega/frailty_graph_viz

An initial wishlist document that can be shared publicly can help understand the “Research” part here

3D Reconstruction with MPSE

rahatzamancse@gmail.com (Rahat Zaman) — Sat, 13 Nov 2021 13:22:57 -0700

The following images are interactive!!!

All of the experiments done for this research are managed by neptune.ai which can be found in this link.

Multi-Perspective Simultaneous Embedding with MDS

A single page quick review of this paper can be found here.

MPSE stands for Multi-Perspective Simultaneous Embedding which is a method proposed by Dr. Stephen Kobourov et. al. for visualizing high-dimensional data, based on multiple pairwise distances between the data points. Specifically, MPSE computes positions for the points in 3D and provides different views into the data by means of 2D projections (planes) that preserve each of the given distance matrices. There are two versions of it: fixed projections and variable projections. MPSE with fixed projections takes as input a set of pairwise distance matrices defined on the data points, along with the same number of projections and embeds the points in 3D so that the pairwise distances are preserved in the given projections. MPSE with variable projections takes as input a set of pairwise distance matrices and embeds the points in 3D while also computing the appropriate projections that preserve the pairwise distances. The proposed approach can be useful in multiple scenarios: from creating simultaneous embedding of multiple graphs on the same set of vertices, to reconstructing a 3D object from multiple 2D snapshots, to analyzing data from multiple points of view.

Using this point embedding algorithm, we have decided to use it for 3D model reconstruction from multiple 2D images. So we propose a novel algorithm to embed image keypoints in a 3D image. The problem assumes a set of image keypoints in an unknown 3D image and several snapshots (2D pictures) of this scene containing these keypoints, either partially of possibly all of the points. Given the coordinates of these keypoints in each snapshot the problem asks to recover their positions in the original 3D image such that the corresponding projections.

There are a lot of 3D reconstruction methods, especially in the field of computer vision with deep learning. But our proposal stands novel and benificial over many ways than deep learning which keep it’s use-case separate from those of deep learning applications. We do not need any supervised training or any pre-existing dataset. As a result, we also do not have the risks that come with deep learning, like overfitting, high training time or data acquisition phase complexity.

3D Reconstruction with MPSE

3D reconstruction using MPSE would have been a straight forward application problem if the perspective images were ideal, e.g. all the points of a single object can be visible in all the images. But it is impossible for all the sides or points of an object to be visible from all the sides. So we had to come up with modifications to the original MPSE algorithm to come up with hidden points (points that are visible from at least 1 viewpoint but not all viewpoints).

Assume we have in total $N$ keypoints that we aim to embed in 3D. However, each of the given view for $1 \le k \le K$ does not include all of these $N$ points. We assume we are given parameters $\alpha_i^k$ for $1 \le k \le K$ and $1 \le i \le N$ which is 1 if the $i$-th points is visible in $k$-th view and 0 otherwise. we can alternatively introduce variables $\alpha_{ij}^k$ which would have a value $1$ if both $i$-th and $j$-th keypoints are visible in $k$-th view and 0 otherwise. The modified objective function for MPSE would be as follows
% The objective function for MPSE with possible hidden points:
\begin{equation}
\sum_{k = 1}^K \sum_{i > j} \alpha^k_{ij} \left( D^k_{i j} - \Vert P^k(x_i) - P^k(x_j) \Vert \right)^2.
\end{equation}
Where $\alpha^k_{ij}$ is either $1$ or $0$. It is 1 when both points $i$ and $j$ are present in projection $k$ otherwise it is 0.
Similarly, if with the use of $\alpha_i^k$, the objective dunctiuon would have the following form:
\begin{equation}
\label{eq:MPSE_hidden_points}
\sum_{k = 1}^K \sum_{i > j} \alpha^k_{i} \alpha^k_{j} \left( D^k_{i j} - \Vert P^k(x_i) - P^k(x_j) \Vert \right)^2.
\end{equation}
where $\alpha^k_{i} = 1$ if the $i$-th point is present and set $\alpha^k_{i} = 0$ if the point is missing.

Our overall contribution in this is shown in the image at the top of this page. Some of the 3D pointsclouds that were reconstructed from only 3 images taken from different positions is given below:

Dataset Requirements

The exact dataset that MPSE needs should have the following features:

A set of perspectives of a pointcloud. A perspective is a set of points of a specific pointcloud transformed or projected onto n-dimension. The goal is to reconstruct the pointcloud in m-dimension where $n < m$.
An annotation of correspondence of each point in different perspectives.

Evaluation

The evaluation of the MPSE algorithm is done based on different parameters of the model and the dataset.

The dissimilarity of two pointcloud is calculated using equation the below equation. Here $x_i$ is the $i$th point of newly generated point cloud and $y_i$ is $i$th point of the ground truth pointcloud.

$$\mathcal{L}(x, y) = \frac{\sum{||x_i - y_i||_2}}{n}$$

The loss function which is optimize For alignment of two pointclouds before calculating the distance is given

\begin{equation}
min_{T}\mathcal{L}(T.x, y)
\end{equation}

Here, $T$ is a 4*4 function and $x$ is the vector of generated point cloud in homogeneous coordinates. $y$ is also of the same form.

After having trails of different optimization functions, we found Adam optimizer to converge the fastest. The whole points alignment algorithm is expressed in the following figure:

Performance on different number of points

Performance on different number of hidden points

Performance on different number of perspective views

This work will be submitted to BMVC

A Note About This Site

rahatzamancse@gmail.com (Rahat Zaman) — Sat, 18 Sep 2021 14:38:58 -0600

About the site

Previously I was using Django to create my personal blog. But as I do not need any kind of server side processing, I thought that django is kind of an overshoot (and also hard to regularly maintain and post compared to many static site generators).

So I started looking for alternatives which are easy to maintain and update. Then I came across the concept of Jamstack and numerous number of static site generators. Among the popular ones, I tried using Gatsby, Jekyll, Hugo, and Hexo. Mike Dane’s channel helped me a lot to get started with these pretty quickly.

Among all the ones I tried, I was impressed by the performance and simplicity of Hugo. I am using it to build this website.

3D Reconstruction with GAN

rahatzamancse@gmail.com (Rahat Zaman) — Tue, 16 Feb 2021 13:22:57 -0700

The following images are interactive!!!
Rotate each of the 3D pointclouds horizontally 90° to see the other view.

The next one is a little complicated with real binary images. The reconstruction is not great, but we can see similarity in the image and perspective easily.

All of the experiments done for this research are managed by neptune.ai which can be found in this link.

Disclaimer

This work is done as a side project of MPSE MDS. Without any duplications, I would suggest to go through that post first before this one.

Presentations

Introduction

Following the original MPSE paper, I tried to implement the algorithm by myself. But, initially I could not get any result with my own implementation, maybe then I did not understood how SGD works with the given stress function. It seems that Stochastic Gradient Descent was not able to minimize the stress function at all (in my implementation).

\begin{equation}
\underset{x \in !R ^{p \times n} }{\text{minimize}} \sum\limits_{k=1}^3 \sum\limits_{i > j} (D_{ij}^k - || P^k(x_i) - P^k(x_j) ||)^2 \
\label{eq:stressmin}
\end{equation}

Later, I started to think about how this can be done in many other ways. I found a new approach which I named mpse-gan.

MPSE-GAN

Primarily, what 3D reconstruction with mpse is doing is generating a set of 3D points that look like a given set of points from the real world from a 2D perspective view. This is exactly what Generative Adversarial Networks can be very good at.

We tried a discriminating neural network with a slightly modified loss function of yours to predict whether a 3D representation presents all the 2D images from some perspectives. The motivation of this came from the Kaggle contest “3D MNIST” where the popular MNIST dataset is given as voxel (volumetric pixels) image and the task is to classify those 3D images. As seen in many example kernels in the contest, deep learning seems to be outperforming most of other optimization models. So, applying deep learning in this context yields high expectations.

Although I did not had enough experience working with 3D images at that time, I have studied about 3D convolution, Computer Vision, Geometric analysis and how to implement those. I have prior experience with GAN which helped me in this research. A theoretical architecture is given in the below figure and a data generation pipeline (IVBA) is tested and proposed on the next figure.

Some examples of the actual implementation is shown at the top of this page.

The actual mathematical explanation can be shown in terms of the original equations with GAN, but with multiple discriminator networks.

\begin{equation}
D = Discriminator
\end{equation}
\begin{equation}
G = Generator
\end{equation}
\begin{equation}
\theta_d = Parameters of discriminator
\end{equation}
\begin{equation}
\theta_g = Parameters of generator
\end{equation}
\begin{equation}
P_z(z) = Input noise distribution
\end{equation}
\begin{equation}
P_{data}(x) = Original data distribution
\end{equation}
\begin{equation}
P_g(x) = Generated distribution
\end{equation}

The Cross-Entropy loss function for the reconstruction process is:

\begin{equation}
\begin{split}
Loss(\hat{y}, y) = y \times log \hat{y} + (1-y) \times log(1-\hat{y})
\end{split}
\end{equation}

So for the whole dataset, it will be:

\begin{equation}
Loss(D(x), 1) = log(D(x)) \
\end{equation}
\begin{equation}
Loss(D(G(z)), 0) = log(1-D(G(z)))
\end{equation}

Objective function of Discriminator networks:

\begin{equation}
\begin{split}
Loss(D)_{min} \longrightarrow max[log(D(x)) + log(1-D(G(z)))]
\end{split}
\end{equation}

Objective function of Generative networks:

\begin{equation}
Loss(G)_{min} \longrightarrow min[log(D(x)) + log(1-D(G(z)))]
\end{equation}

So Overall Objective function will be:

\begin{equation}
min_G max_D [log(D(x)) + log(1 - D(G(z)))]
\end{equation}

Iterations

At each of the iterations, we can see how the reconstruction is actually happening at each iteration.

You can click on each image to zoom in.

All the corresponding loss values at each iteration can be found in the neptune experiment website.

This work will be submitted to BMVC

Machine Learning Tutorial - Lesson 00 (Overview)

rahatzamancse@gmail.com (Rahat Zaman) — Sat, 12 Dec 2020 00:00:00 +0000

All these tutorial are written by me as a freelancing working for tutorial project AlgoDaily. These has been slightly changed and more lessons after lesson 12 has been added to the actual website. Thanks to Jacob, the owner of AlgoDaily, for letting me author such a wonderful Machine Learning tutorial series. You can sign up there and get a lot of resources related to technical interview preparation.

Whole Series Website

Machine Learning Series structure

In this page, I am going to introduce to machine learning basics over 12 lessons. Unlike all other tutorials, I will not explain how important it is to learn machine learning. If you are here, you already know about it and want to dig straight into how to do it. So let’s get started. The overall structure of this tutorial is given below.

Lesson 1: Introduction to Machine Learning

What is Machine Learning
Prerequisites.
Brief overview of Python
10 Example Applications of ML
Core packages that will be covered in the series: Python, Numpy, OpenCV, Scikit-Learn, Scikit-Image, Tensorflow & PyTorch
Warming up Python Skills
Jupyter Notebook, Python Setup, Conda, Virtualenv
Finally end with a fun linear regression model from scratch.

Lesson 2: All about Numpy and Pandas

Matrix and Data
Dimensions and Shapes of numpy ndarray.
Broadcasting in numpy
Pandas and Numpy differences
Using lambda on numpy and pandas.

Lesson 3: Supervised learning and unsupervised learning; Instance based learning and object based learning

Data and their Labels
Batch and Online learning
Linear Regression
Dataset divided into train and test

Lesson 4: Machine Learning basic from scratch

Hyperparameter tuning and Model Selection
Linear Regression from scratch
Gradient Descendent Algorithm

Lesson 5: Statistics in Machine Learning

Matplotlib
Plotly

Lesson 6: Data Visualization

Probabilistic Model
Data Distributions
Data Preprocessing
Normalization and Standardization
Hypothesis Testing (student’s T-test)

Lesson 7: All about Scikit-Learn (Supervised Learning only)

Binary/Multiclass Classification, Detection, Recognition etc.
Logistic Regression, Linear Regression, Polynomial Regression, Activation Function etc.
SVM, Decision Tree, Ensemble learning, Random Forest

Lesson 7.5: All about Scikit-Learn (UnSupervised Learning only)

Dimensionality, PCA, t-SNE, KNN etc.
K-means, Clustering, DBSCAN etc.

Lesson 8: Neural Networks Basics

Lesson 9: Introduction to TensorFlow

Lesson 10?: Advanced Deep Learning Networks

Image Processing
OpenCV
CNN
Finding dataset
Finding models

Lesson 11: Trying out popular Deep Learning Models

Popular models and their implementation in python
- Yolo
- Inception
- Residual Net
- Alex NET
- GANs
GTP3

Lesson 12: Real world ML problem solve exercise

You Can Make Anything With Rofi

rahatzamancse@gmail.com (Rahat Zaman) — Fri, 21 Aug 2020 18:24:29 +0600

What is rofi?

Rofi is a popup list displayer which will display a list of whatever you want and do whatever you want with the selection you make from the prompted list.

Rofi started as a clone of simpleswitcher, written by Sean Pringle - a popup window switcher roughly based on superswitcher. Simpleswitcher laid the foundations, and therefore Sean Pringle deserves most of the credit for this tool. Rofi (renamed, as it lost the simple property) has been extended with extra features, like an application launcher and ssh-launcher, and can act as a drop-in dmenu replacement, making it a very versatile tool.
– Rofi - GitHub

Rofi currently supports only X server. But you can find some forks of Rofi making them support wayland such as this one.

Some use cases of Rofi

Below I am proving some functionalities I use regularly with the help of Rofi. You can find all my dotfiles related to rofi in this link

Manage your configuration files

I use the git bare repository method to manage my dotfiles which is explained here. To manager your dotfiles, you have to create a folder where your .git directory will stay. After that, type the three commands below. I have considered my .git directory to be in ~/.dots-git directory.

1
2
3
4
5
6


# Initialize the repository
git init --bare $HOME/.dots-git
# make an alias to convenience (put this in your init script like bashrc/zshrc file)
alias dotfiles='/usr/bin/git --git-dir=$HOME/.dots-git/.git --work-tree=$HOME'
# Untrack files from your working directory
dotfiles config --local status.showUntrackedFiles no

After doing this, you can add a config file to manage with git with the following command.

1
2


dotfiles add FILE_NAME
dotfiles commit -m "Updated FILE_NAME and added THIS function"

You can also put your configs to a repository online by setting up a remote. Google if you don’t know how to manage git repositories.

In a fresh installation, you can download and run the below command from an online gist and all your settings will come back to the new machine!

The below script will overwrite your current settings.

1
2
3
4
5
6


cd $HOME
mkdir -p .dots-git/git-control/.git
/usr/bin/git clone YOUR_GIST_LINK --bare .dots-git/git-control/.git
alias dotfiles='/usr/bin/git --git-dir=$HOME/.dots-git/git-control/.git --work-tree=$HOME'
dotfiles checkout -f
echo "DONE"

Okay, as now you know how to manage your dots with git, lets setup rofi to quickly see and open your managed dotfiles. Below is the script:

1
2
3
4
5
6
7
8
9


#!/bin/bash

# Options
DOTS_GIT=$HOME/.dots-git/git-control/.git

dotfile_paths=$(/usr/bin/git --git-dir=$DOTS_GIT --work-tree=$HOME ls-tree -r master --name-only)

chosen="$(echo -e "$dotfile_paths" | rofi -dmenu -no-custom)"
[ -f "$chosen" ] && $TERMINAL $EDITOR $chosen || dunstify "No such file"

The main idea behind this is the command dotfiles ls-tree -r master --name-only which will print all the files in your git repository. The rest is pretty much simple to understand.

Use rofi as OCR

I am currently learning Japanese. So I often need to see the meaning of an article from a book written in English or Japanese. For this, I use tesseract and its language model dependencies. You will also need scrot to take screenshots of your desired portion of screen and imagemagick for some image processing. If you are using Arch Linux, go ahead and install them with one liner.

1

yay -S tesseract tesseract-data-eng tesseract-data-jpn tesseract-data-ben scrot

Then, all you need is to write a small script and assign it a keybinding.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26


#!/bin/bash
# DEPENDENCIES: tesseract-ocr imagemagick scrot

APP_NAME="Screen Reader"

export LC_ALL=en_US.UTF-8

LANG="$(echo -e "eng\nita\ndeu\nben\njpn" | rofi -dmenu -no-custom)"
if [ -z "$LANG" ]; then
 dunstify -a "$APP_NAME" "Invalid Language"
 exit 1
fi

echo "Language set to $LANG"

SCR_IMG=$(mktemp) # create tempfile
trap "rm $SCR_IMG*" EXIT # make sure tempfiles get deleted afterwards

#take screenshot of area
scrot -s "$SCR_IMG".png -q 100
# postprocess to prepare for OCR
mogrify -modulate 100,0 -resize 400% "$SCR_IMG".png
# OCR in given language
tesseract -l "$LANG" "$SCR_IMG".png "$SCR_IMG"
xsel -bi < "$SCR_IMG".txt # pass to clipboard
dunstify -a "$APP_NAME" "Text Copied to clipboard"

I have used dunstify to display nice notification when the OCR is done or the language could not be detected. You can comment those lines out if you do not wish to use it.

This script will first prompt the language you want to do OCR on, then start scrot screen clipping mode. Then it will do the ORC and copy the text in your clipboard.

Super useful plugins

Rofi is already mighty by itself as it can display a list to run any application in from you desktop files or commandline programs. It can also switch focus to a open window regardless of which workspace/desktop the window is in, as long as your window manager implements Window Manager Specification Project. But to enhance its functionality beyond just a application launcher, you can add tons of plugins taken from the internet (or make one yourself). Below are the plugins I use in my daily driver machine most of which are taken straight from the AUR. Below is the image of my main rofi interface.

I achieved this by using the following launch script and some theming with rasi.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12


#! /bin/sh

rofi \
 -modi combi,file-browser,"CALC:qalc +u8 -nocurrencies","CLIPBOARD:greenclip print",run \
 -show combi \
 -combi-modi window,drun \
 -display-combi APPS \
 -display-drun "" \
 -display-window "󰖯" \
 -display-file-browser FILES \
 -display-run SHELL \
 -columns 2 \

1. rofi-calc

Bring the calculator up straight hitting a shortcut key. And don’t think this lightweight calculator will do just some addition and substitution. It can do advanced calculations like the below one.

> solve(x*x/pi = 3 acres) to km
solve(((x * x) / pi) = (3 * acre), x) = 6.1758176 km
> fibonacci(133) to hex
fibonacci(133) = 0x90540BE2616C26F81F876B9
> 50m*2kg/50h to ?N
(50 meter (2 kilogram)) / (50 hour) = 555.55556 μNs

It can do this, because it uses libqalculate as the backend.

2. rofi-dmenu

A lot of program sometimes use dmenu for querying user or display things. If you have rofi installed, there is actually no reason to keep dmenu just for those applications because rofi can do everything dmenu does. But, well, the programs will look for dmenu anyways. So you can just manually create an alias for rofi -dmenu to use rofi as an in-place replacement for dmenu. Or, what I do is install this Arch package which does exactly this.

3. rofi-file-browser-extended

Yes, Rofi can be used as a file browser. If you want to quickly open up a file from not so deep directory in you home, you do not need to open up a heavy file manager (e.g. Nautilus/Thunar). Just use super + space (or whatever you like) to open up file browser and open a file. But as the name says, it is just a file browser, not a file manager. So you cannot copy/cut/paste your files or do such like things.

4. rofi-greenclip

Greenclip is a simple clipboard manager which has functionality to save your clipboard and manage them in your own way. Rofi-greenclip is an extension of rofi for the interfacing of greenclip. You can see your clipboard history and copy any text among those.

5. rofi-search

Rofi-search is a interactive web searching shortcut supporting Google and Duckduckgo. You can just search first from rofi and then open up your desired website only on your favorite browser. I did not include this with my default rofi access menu because it serves a bit different purpose than doing things in the desktop. So it has a separate keybinding in my machine. I use the below script to use in a keybinding.

1

GOOGLE_ARGS='["--count", 10]' rofi -modi blocks -blocks-wrap rofi-search -show blocks -lines 5 -eh 4 -kb-custom-1 'Control+y'

6. rofi-top

Rofi-top shows top in your rofi. Thats it. I have put a keybinding like this in my sxhkdrc.

1
2


super + r; t
 rofi -modi top -show top -eh 1 -kb-custom-1 'Alt+m' -kb-custom-5 'Alt+c'

As you can see, I can clearly tell that my google-chrome is eating up all my memory for opening 4 tabs 😄 alongside the VSCode behind this picture.

7. rofimoji

The last extension is what I use the most while writing an article or code. I have also specified a separate keybinding for this extension. Whenever I need an emoji, I search it in rofi and rofi automatically takes it into the clipboard and also puts one emoji of that into the focused text area.

Conclusion

I have read in the internet someone sarcastically saying to just use rofi to make a desktop environment. But to be honest, with some major tweaking of the code, it is not much far from impossible. If you use rofi in a cool way, please share it in the comments so I also can use it.

A Trivial Visualization with Plotly Express

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 20 Aug 2020 00:00:00 +0000

This post demonstrates a trivial way to perform data analysis and extract useful insights. This example shows some effective analysis using rigorous data visualization, hypothesis testing to solve a prediction problem, and building regression/classification model to learn something interesting.

I will be using python because it is the most comprehensive way for data analysis in my opinion.

Python modules used for this posts:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


ipython
ipywidgets
jupyterlab==2.2.6
matplotlib==3.3.2
numpy==1.19.1
pandas
plotly==4.12.0
scikit-learn
tensorflow==2.3.1
tqdm

Dataset description

The dataset used for this post contains the following information:
Raw accelerometer sensor data is collected from the smartphone and smartwatch at a rate of 20Hz. It is collected from 29 test subjects as they perform 6 activities (walking, jogging, ascending stairs, descending stairs, sitting, and standing) over a span of time.

Attribute Information

subject-id: ID describing the participant
activity-code: timestamp: Unix time (integer)
x: represents the sensor reading (accelerometer) for the x dimension
y: represents the sensor reading (accelerometer) for the y dimension
z: represents the sensor reading (accelerometer) for the z dimension

The provided dataset is a simple text file where each line represents a data point having the above attributes. Below is a small portion from the first line of the file.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18


33,Upstairs,49591302292000,0.38136974,13.865514,1.2258313;
33,Upstairs,49591412308000,-2.1111538,10.147159,0.88532263;
33,Upstairs,49591522323000,0.38136974,5.366417,0.6537767;
33,Upstairs,49591632309000,1.1849703,4.0180025,3.5957718;
33,Upstairs,49591742294000,1.334794,17.011814,4.0588636;
33,Upstairs,49591852310000,3.568531,11.073342,0.6946377;
33,Upstairs,49591952316000,1.1849703,6.742072,0.10896278;
33,Upstairs,49592052292000,0.6537767,7.1234417,0.6946377;
33,Upstairs,49592112228000,1.5390993,8.076866,0.313268;
33,Upstairs,49592222305000,4.3312707,13.102775,0.95342433;
33,Jogging,49105962326000,-0.6946377,12.680544,0.50395286;
33,Jogging,49106062271000,5.012288,11.264028,0.95342433;
33,Jogging,49106112167000,4.903325,10.882658,-0.08172209;
33,Jogging,49106222305000,-0.61291564,18.496431,3.0237172;
33,Jogging,49106332290000,-1.1849703,12.108489,7.205164;
33,Jogging,49106442306000,1.3756552,-2.4925237,-6.510526;
33,Jogging,49106542312000,-0.61291564,10.56939,5.706926;
...

Libraries used

So first things first, we will import all the necessary modules we need. Often this part starts from blank, slowly growing as we start feeling the need of a specific task and import related modules.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29


# Basic data manipulation
import math
import numpy as np
import pandas as pd

# Nice progress bar
from tqdm import tqdm_notebook as tqdm

from tensorflow.keras.utils import to_categorical # convert to one-hot-encoding

# For Data preprocessing
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

# For a learning model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, LSTM
from tensorflow.keras import regularizers

# Suppress warnings
import warnings
warnings.filterwarnings('ignore')

# For visualization in jupyterlab
import plotly.io as pio
pio.renderers.default = "jupyterlab"
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff

Load the data into memory

First things first, we need to load to data into memory to do futher processing. This part is pretty easy if there is a very small size single file. But this part could be very complex in case of large dataset. You might have to load the data in chunks (pandas has option for that), or take samples from the whole dataset to fit it into memory.

I prefer to do this step in an if-else block. I keep a boolean variable named FAST_LOAD initially defaulted to False at the top of the code. After some trial and errors with the loading of the data and finalizing the loading section, I save a kind of “snapshot” of the loaded data into disk so that I can easily load the data quickly from next time. Then I make the FAST_LOAD variable to true. Next time I run that code cell, the data gets loaded instantly, no matter how large it was in the first place.

The train.txt file does not seem to be a common csv file that we could load with pandas. CSV files usually end with a newline at each row. But this one has ; (semicolon). After reading the data as string, we can replace the ; with \n and then try to load with pandas as csv. Below is the cell:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27


FAST_LOAD = True

if FAST_LOAD == False:
 with open('train.txt', 'r') as f:
 data = f.read().replace(';','\n')

 with open('train_proc.txt', 'w') as f:
 f.write(data)

 df = pd.read_csv(
 'train_proc.txt',
 header=None,
 sep=',',
 names=['id', 'activity', 'timestamp', 'x', 'y', 'z'],
 skip_blank_lines=True,
 )

else:
 df = pd.read_csv(
 'train_proc.txt',
 header=None,
 sep=',',
 names=['id', 'activity', 'timestamp', 'x', 'y', 'z'],
 skip_blank_lines=True,
 )

df.head()

1
2
3
4
5
6


 id activity timestamp x y z
0 33 Upstairs 49591302292000 0.381370 13.865514 1.225831
1 33 Upstairs 49591412308000 -2.111154 10.147159 0.885323
2 33 Upstairs 49591522323000 0.381370 5.366417 0.653777
3 33 Upstairs 49591632309000 1.184970 4.018003 3.595772
4 33 Upstairs 49591742294000 1.334794 17.011814 4.058864

As our dataset is in a well-organized pandas DataFrame, we can get a lot of information with the help of pandas library!

1

df.describe()

1
2
3
4
5
6
7
8
9


 id timestamp x y z
count 1.098081e+06 1.098081e+06 1.098081e+06 1.098081e+06 1.098080e+06
mean 1.886074e+01 3.340610e+13 6.625116e-01 7.255568e+00 4.111336e-01
std 1.021471e+01 4.945179e+13 6.849173e+00 6.745983e+00 4.754197e+00
min 1.000000e+00 0.000000e+00 -1.961000e+01 -1.961000e+01 -1.980000e+01
25% 1.000000e+01 2.018902e+12 -2.870000e+00 3.170000e+00 -2.220000e+00
50% 1.900000e+01 9.719432e+12 2.700000e-01 7.930000e+00 0.000000e+00
75% 2.800000e+01 4.995661e+13 4.440000e+00 1.156000e+01 2.720000e+00
max 3.600000e+01 2.093974e+14 1.995000e+01 2.004000e+01 1.961000e+01

1

df.info()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12


RangeIndex: 1098081 entries, 0 to 1098080
Data columns (total 6 columns):
 # Column Non-Null Count Dtype
--- ------ -------------- -----
 0 id 1098081 non-null int64
 1 activity 1098081 non-null object
 2 timestamp 1098081 non-null int64
 3 x 1098081 non-null float64
 4 y 1098081 non-null float64
 5 z 1098080 non-null float64
dtypes: float64(3), int64(2), object(1)
memory usage: 50.3+ MB

Data Cleanup Before Visualization

We should not go directly to visualization before at least being sure that the visualization code will work properly. For example, a single null value the dataset will throw an error while trying to visualize it with matplotlib. Fortunately, plotly tries to handle most of these problems for use. But still this is always a good practice to make sure that the data does not have any extremely high or low value, or in some cases, zero values.

1
2
3


# Make sure there is no zeros
df = df[(df['x'] != 0) & (df['y'] != 0) & (df['z'] != 0)]
df[(df['x'] == 0) & (df['y']==0) & (df['z']==0)].count()

1
2
3
4
5
6
7


id 0
activity 0
timestamp 0
x 0
y 0
z 0
dtype: int64

All rows containing zeros are gone. Now let us check for null values.

1
2
3
4
5
6


# Checking missing data
total = df.isnull().sum().sort_values(ascending=False)
percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)

missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.head(20)

1
2
3
4
5
6
7


 Total Percent
z 1 9.296592e-07
y 0 0.000000e+00
x 0 0.000000e+00
timestamp 0 0.000000e+00
activity 0 0.000000e+00
id 0 0.000000e+00

We can see that there are some null values in the z column. We can remove that easily. Withing removing them, there are other options like replacing the value with average or median, but in this context, the nulls are very small, so removing them will not hurt that much.

1

df.dropna(inplace=True)

Data visualization

Let us start with the class imbalance property check. This is to check if the data is equally divided to all the target column values (in this case activity column). Below is the code to check it on the dataframe.

1
2
3
4
5
6
7
8
9


categorical = True
target_col = 'activity'

if categorical: # Histogram is preferred if target is discrete
 fig = px.histogram(df, x=target_col)
else:
 fig = ff.create_distplot([df[target_col]], group_labels=[target_col], curve_type='normal')

fig.show()

There is class imbalance problem in the dataset. Deep learning based approaches usually do not work well with class imbalanced dataset. The easiest way to solve this problem is by sampling the data while training (but often this is not enough, because it can overfit the model to some classes).

1

df_sampled = df.groupby('activity', group_keys=False).apply(lambda x: x.sample(min(len(x), 200)))

Now to determine the difference in the sparsity of the data, we can create a scratter plot. For 2D and 3D data (this dataset has 3 numeric attribute: x, y, z), we can directly plot the data. But when the data has more that 3 dimensions, we either need to plot a part of the dimensions, or we can use a dimension reduction method like PCA (Principles Component Analysis).

1

px.scatter_3d(df_sampled, x='x', y='y', z='z', color='activity')

Now for trainning a model, we need the data attributes (usually named X) and their corresponding labels (usually named y).

Time series Data

Because the given data is a very long time series data, we can create a window of length time_len. Then, we can sample time_len consecutive data-points (x, y, z) to get single windows from the dataset. Although we can do this as much as we want, but a good way to is sample from the data about less then the actual size. For example, if you have a dataset with n points, and time_len = 20, then you should sample less than n/time_len time. Else, there will be too much duplicate data for the model.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28


WRITTEN = False

if WRITTEN == False:
 X = []
 y = []
 time_len = 20

 c = 0
 for i in tqdm(range(0, len(df)-time_len-1, time_len)):
 sample = df[i:i+time_len]

 if sample['activity'].values[0] == sample['activity'].values[time_len-1]:
 X.append(sample[['x', 'y', 'z']].values)
 y.append(sample['activity'].values[0])
 else:
 c += 1
 print('Numer of skipped :', c)

 X = np.array(X)
 y= np.array(y)
 np.save('X_train', X)
 np.save('y_train', y)

else:
 X = np.load('X_train.npy')
 y = np.load('y_train.npy')

print("Total trainable windows: ", len(X))

We can see a sample Data-Point from the (X, y) pair.

1

print(X[10], y[10])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


[[ 3.8273177 18.04696 6.851035 ]
 [ 0.08172209 -7.8861814 -8.19945 ]
 [ -1.525479 17.352324 -1.8387469 ]
 [ -1.4573772 14.056199 2.982856 ]
 [ -6.701211 4.5628166 3.1054392 ]
 [ -1.56634 1.2666923 0.88532263]
 [ -0.95342433 19.57244 8.349273 ]
 [ -0.9125633 -3.9499009 -2.982856 ]
 [ -2.0294318 11.413852 15.4046135 ]
 [ 7.8589406 18.700737 6.0882955 ]
 [ -1.1441092 16.548723 6.742072 ]
 [ 4.1814466 3.8681788 -0.14982383]
 [ -3.5276701 18.087822 -3.8681788 ]
 [ 5.6252036 9.084772 6.6331096 ]
 [ 2.2201166 -5.8158884 -10.106298 ]
 [ -9.193735 3.568531 -6.238119 ]
 [ 3.173541 15.241169 1.6480621 ]
 [ 5.284695 3.405087 1.2258313 ]
 [ 1.4573772 10.501288 -1.4165162 ]
 [ 3.5276701 19.19107 4.903325 ]] Jogging

Now, we can select any window from X and plot that as a time series line plot to get a visual representation of each sample window. The first window samples from each activity label (“Sitting”, “Standing”, “Walking”, “Jogging”, “Downstairs” and “Upstairs”) is visualized below.

Sitting

Standing

Walking

Jogging

Downstairs

Upstairs

Observation from the visualization

One can clearly distinguish between the difference of the data according to their labels. More heavy tasks (like jogging and going upstairs) is far more spiky than easier tasks (like sitting and standing). On the other hand, going downstairs and upstairs are very similar in spikes.

But, if you look closely, for the diagram of going upstairs, the values of y (visualized in red) is most of the time higher than the values of z (visualized in green). And in case of going downstairs, the values of z (again visualized in green) is most of the time higher than the values of y (visualized in red). This can be learned by any modern machine learning classifier.

Conclusion

This is it for the visualization part. Later in another post, I will continue on this dataset to train different models and see which works better and why.

The dataset is provided by the North South University ACM SC for the Datathon segment of a Computer Science Festival named Technovation 2.0 held at the premises of North South University. My team, KUET MANJARO consisting of 2 members, become the runners up of the event among 30+ national teams from several universities consisting of 4 members.

The champion of the first-ever university-level Datathon in Bangladesh was Team HardMax from North South University, with the Runner-Up KUET_Manjaro from KUET.
– Official Website of NSU

Some Notes About SXHKD

rahatzamancse@gmail.com (Rahat Zaman) — Wed, 19 Aug 2020 22:22:50 +0600

Simple X Hotkey Daemon

In simple words, SXHKD is a keyboard shortcut manager for any X server. It runs as a process or daemon and grabs keys to call bash commands. This is created by the same authors of BSPWM, the popular window manager.

Some people think that, SXHKD is/works only for BSPWM. But this is just a myth. SXHKD will work with any window manager or even desktop environments like Gnome as long as it is running the X window server.

This post contains just some advanced usage of SXHKD. So without writing any introductory information about the software as you will find tons of things better than I will ever write, I am writing just some cool aspects you can achieve with SXHKD that is hardly found on the internet.

My sxhkd setup

I have been using BSPWM for 1 year only and I am in love with this amazing software sxhkd. I use this to manage windows and launch frequently used programs like terminal/browser/file-manager. The way I use sxhkd is vim-like multimodal window editing.

When I am in normal mode, I use super + {h,j,k,l} to move focus to other windows. To move window to another place, I use super + shift + {h,j,k,l}. And when I go to VISUAL-MODE, I use {h,j,k,l} to move focus to other windows. To move window to different location while in VISUAL-MODE, I use super + {h,j,k,l}. Though it is totally my personal preference, I think this way, it saves a lot of key combos to do advanced layout operations like flipping/rotating layouts etc.

You can find my sxhkdrcs’ in my linux-dots repository.

Without further ado, let me just share the things I am able to achieve.

VIM like multi-mode with polybar status

You can change modes of your keyboard in SXHKD just like you change mode in vim!

Notice that, sxhkd has built in support for modes (called key chords). But they have some limitations, like you cannot define how to switch between chords or exit from one. Also, it becomes cluttered if you dump all configurations of all modes in a single sxhkdrc file. So I went to a different approach.

To change mode with a keybinding, you need to restart the sxhkd with different config file. So select a key comb you want to use to change to a specific mode and put the following lines in your sxhkdrc file.

# Change to VISUAL mode
super + m
killall -e sxhkd && sxhkd -c $HOME/.config/sxhkd/mode-visual &

Here, mode-visual is another sxhkdrc file with the bindings necessary for that new mode in sxhkdrc. To go back to the default or other mode, just do the same with your desired mode’s config file.

# Exit visual mode
@Escape
killall -e sxhkd && sxhkd &
super + m
killall -e sxhkd && sxhkd &

As shown in the above gif, you can use polybar hook to update an icon indicating the current mode of sxhkd.

# Change to VISUAL mode
super + m
killall -e sxhkd; polybar-msg hook window-mode 2 > /dev/null && sxhkd -c $HOME/.config/sxhkd/mode-visual &

And in your polybar configuration, use the below module in any bar you want:

1
2
3
4
5
6


[module/window-mode]
type = custom/ipc
content =
hook-0 = echo "󰍹"
hook-1 = echo %{F#f00}󱋆%{F-}
initial = 1

I basically use this just like vim. In normal mode, I can use the current window and navigate to other windows normally. But in VISUAL-MODE I can more advanced window moving like rotating/flipping layouts, adjusting sized and receptacles, etc. This way you can save a lot of keybindings that you do not need while simply using 2 or 3 windows.

Organize and use sxhkd in multiple setups

I do not remember where I found this on internet, but with sxhkd, you can use multiple config files at the same time overriding same keybindings by the latter loaded config file. For example, using the below command:

1

sxhkd -c $HOME/.config/sxhkd/sxhkdrc $HOME/.config/sxhkd/sxhkdrc.common

will first load the sxhkdrc file, and then load sxhkdrc.common file. If they have a common keybinding in both files, sxhkdrc will be given priority.

By this way, you can put all your machine independent keybindings in sxhkdrc.common file, and put machine specific keybindings in sxhkdrc file of that machine.

Display keyboard shortcut help with rofi/dmenu

This one looked very cool to me. You can easily parse your sxhkdrc file and display them as a help menu with dmenu or rofi. Use the below script for this. You can put this in show-help script in a directory which is in your PATH.

1
2
3
4
5


#!/bin/bash

awk '/^[a-z]/ && last {print "<small>",$0,"\t",last,"</small>"} {last=""} /^#/{last=$0}' ~/.config/sxhkd/sxhkdrc{,.common} |
 column -t -s $'\t' |
 rofi -dmenu -i -markup-rows -no-show-icons -width 1000 -lines 15 -yoffset 40

Your sxhkdrc file should be well documented like the following structure.

# Your Documentation for this keybinding
super + a
your_command

Just put a keybinding to show help for all keybindings 😲

# Show help
super + slash
sxhkd-help

Conclusion

I really like the software sxhkd because it goes with the unix philosophy. It is straight forward and very much customizable. With the help of xdotool, you can pretty much do whatever you want!

If you also have dome some cool things with sxhkd, I will be glad to hear some in the comments!

My Polybar Tips and Tricks

rahatzamancse@gmail.com (Rahat Zaman) — Tue, 18 Aug 2020 09:43:24 +0600

Why choose Polybar

Trust me, there are open source bars in the world in a number you can’t even imagine. Also, you can create your own bar instead! (OldTechBloke has a good demonstration).

Lemonbar
i3 bar
Conky (yes, you can use this as a bar also)
Candybar
Dzen
Tint2
n30f
Cairo-dock
xmobar

I have just taken a look at lemonbar and i3 bar and found that polybar is best suited for me. Below are the reasons I like polybar the most.

Very easy to configure. You do not need to know any language (but knowing how .ini files work will be a bonus).
Is extensible with IPC (Inter Process Communication). So you can put any command’s output or trigger a thing for any event in the OS.
The built in modules are very useful and may be all that you need to put into a bar.
The maintainers are great! Put any issue you want and they’ll respond very fast.
You can choose your font and customize it’s size, offsets, thickness etc. to give it the look and feel you want.

My polybar setup

My polybar configuration can be found on my dots repository. This setup is highly inspired by the famous post in r/unixporn by juker97.

Below is an image of my complete polybar setup which I use regularly.

My complete desktop:

The directory structure I maintain in my ~/.config/polybar is given below:

~/.config/polybar/
├── scripts/
│ ├── corona.sh
│ ├── cpu.sh
│ ├── power-menu.sh
│ ├── power-menu.rasi
│ └── ...
├── autostart
└── config

As you can see, the default location of the config file of polybar is kept same. I keep all the scripts related to polybar in the scripts folder. The autostart script is a bash/zsh script I use to run multiple polybars with only one command (that is running the script only)

My autostart file is something like this.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17


#!/bin/sh

# Terminate already running bar instances
killall -q polybar

# Wait until the processes have been shut down
while pgrep -u $UID -x polybar >/dev/null; do sleep 1; done

# Create gap for polybar
bspc config bottom_padding $(($(bspc config bottom_padding)+35))

# Start all the bars
polybar launcher &
polybar workspace &
polybar systray &
polybar status &
polybar power &

The below tips and tricks are written in such a way that it assumes you have basic knowledge of how linux and polybar works. If you do not know what polybar is or do not know what linux environments are and how they work, then go ahead and learn those first. You’ll find tons of tutorial and documentations which will be thousands times better than a tutorial I will ever write.

The Tips

1. Kill all polybar instance before creating a new instance.

Whenever you are starting/restarting your polybar instance, it is a good idea to kill the previous instance first. To do that, you can run the following command to kill all the instances at once.

1

killall -q polybar

Sometimes, you may need to wait for the instance to kill before starting the new instance. Run the following command to wait for that moment.

1

while pgrep -u $UID -x polybar >/dev/null; do sleep 1; done

This might sometimes wait forever if killall -q polybar could not kill all instances of polybar correctly. Though it never happened to me.

2. Show polybar in the desired monitor you want

Polybar has an option in config file named monitor which can be used with a bar. For more details about it, refer to Bar settings section in the polybar wiki. I am adding this because before seeing in the page, I sort of accepted that polybar bars can only be in the primary monitor.

Below is a simple example:

1
2


[bar/abar]
monitor = YOUR_MONITOR_NAME_HERE

3. Use the IPC as less as possible

The IPC (Inter Process Communication) is one of the most attractive features of Polybar. Basically what it does is it waits for another outside process to trigger a hook stored in the polybar and run a bash command (sort of like a daemon). But it takes comparatively a lot of CPU to keep listening for a hook to trigger. Most of the built-in modules will do what you want to achieve with a very low resource requirement. So use the IPC if you absolutely need to use.

4. You can use the Xresources in your polybar

To match the color scheme of all the desktop applications (and wallpaper), a lot of people use environment variables or Xresources to store and use their color schemes. Many use Base16 color schemes in their entire rice.

Polybar has its own configuration file which is not bash or any programming language. You have the option just to use it to do whatever you want. Good news is, polybar has built-in support for Xresources with the help of xrdb. Below is an example how you can load the color scheme in polybar config.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


[colors]
; wpgtk colors
background = ${xrdb:color8:#FF1C2C43}
background-alt = ${xrdb:color0:#FF0e1827}
foreground = ${xrdb:color15:#FEFEFE}
foreground-alt = ${xrdb:color9:#9ac4ff}
primary = ${xrdb:color10:#fff}
secondary = ${xrdb:color1:#a9b3c2}
alert = ${xrdb:color9:#bd2c40}

; Use these ase ${colors.background}

I may be telling polybar config as an ini file, but it is not. But they look very similar to each other.

The Tricks

1. Dynamically select monitor to display polybar

Polybar config can read environment variables with the help of special syntax. You can take any environment variable with ${env:VAR_NAME:}. So, you can take a variable with the name of your desired monitor, and use it inside polybar with the monitor = ${env:MONITOR:} syntax. You can use this with the help of xrandr or other wrappers of xrandr (like arandr/autorandr) to send polybar to desired monitor.

The below example will be very useful to you if you are using a laptop and changing your attached monitors setups frequently.

These codes will do the following: If there is a second monitor found, start polybar in that monitor, else fallback to run the polybar bar in the primary monitor. You can also adjust more granular components (like height and width depending on the monitor resolution) with more environment variables.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16


[bar/topright-extras]
monitor = ${env:MONITOR:}

width = 400
height = 25
fixed-center = true
bottom = false

background = ${colors.background}
foreground = ${colors.foreground}
border-color = #000000

font-0 = BreezeSans:style=Medium Condensed
font-1 = Material Design Icons Desktop

modules-left = corona

To run this config with the proper monitor, use the following script:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12


eval $($HOME/.local/bin/get-mons.sh)

bspc config -m $MONITOR_1 bottom_padding $(($(bspc config bottom_padding)+35))

if [ ! -z $MONITOR_2 ]; then
 bspc config -m $MONITOR_2 bottom_padding $(($(bspc config bottom_padding)+45))
 MONITOR=$MONITOR_2 polybar topright-extras &

# Fall back to ealier monitor
else
 MONITOR=$MONITOR_1 polybar topright-extras &
fi

Here, get-mons.sh is a script which sets some variables about monitors:

1
2
3


#!/bin/sh

xrandr --listmonitors | awk 'BEGIN{count=0} NR>1 {count=count+1;split($3,a,"/");split(a[2],b,"x");print "MONITOR_"count"="$NF"\nHEIGHT_"count"="b[2]"\nWIDTH_"count"="a[1]"\n"}'

The output of get-mons.sh is something like below:

1
2
3
4
5
6
7


MONITOR_1=HDMI1
HEIGHT_1=1080
WIDTH_1=1920

MONITOR_2=eDP1
HEIGHT_2=768
WIDTH_2=1366

2. Create a double column status in polybar module

Ever wondered how you can achieve this with your polybar?

You need some font customization to get this. This will work with any font you want!

Firstly, select your favourite font. Then use the same font twice in polybar config just with different offset values like below.

1
2
3
4


[bar/status]
font-0 = BreezeSans:pixelsize=10:style=Bold Condensed;-7
font-1 = BreezeSans:pixelsize=10:style=Bold Condensed;10
; ...

Then use the fonts in your module.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16


[module/wlan]
type = internal/network
interface = ${env:PRIMARY_NETWORK_ADAPTER}
interval = 3.0

format-connected =<ramp-signal><label-connected>
label-connected = %{T1}%upspeed%%{T-}%{O-35}%{T2}%downspeed%%{T-}

label-disconnected = 󰤭
label-disconnected-foreground = #6c809e

ramp-signal-0 = %{F#ff004b}󰤫%{F-}
ramp-signal-1 = %{F#ffd200}󰤟%{F-}
ramp-signal-2 = 󰤢
ramp-signal-3 = 󰤥
ramp-signal-4 = 󰤨

3. Do not allow polybar to cut a portion off the screen.

Sometimes you want to manage the screen space by yourself. But polybar usually takes a space if it recognizes the window manager. For example, if you use BSPWM and enable wm-restack = bspwm in your settings, then windows will leave a space at the top/bottom where you polybar lives. This will also keep the polybar always on top except full screen mode.

You can avoid this polybar’s screen space taking by using the following in your global section of polybar config. This will make sense to those who are familiar with CSS box model.

1
2
3


[global/wm]
margin-top = -100%
margin-bottom = -100%

Conclusion

So those are my most helpful tips and tricks to share with all. If you have any other good ideas, please let me know in the comments. Also ask any question about anything you might want to know in more details.

Switching from VIM to NeoVIM

rahatzamancse@gmail.com (Rahat Zaman) — Sat, 25 Jul 2020 12:52:00 +0600

What I use VIM for?

I am not a big fan of “Use everything in Terminal”. Not because I am not comfortable with terminals and TUI, but I understand and miss the lacking features of GUI in a TUI in this twenty first century. Let’s talk about the details and causes for this on some other post for now.

I use VI/VIM only when I had to edit a file quickly and I am in a terminal for some work. So, for this reason, I do not need to turn VIM into an IDE like IntelliJ of Visual Studio. All I want is a very simple editor with just the basic information I need to make my editing comfortable while I am in a terminal.

So, this are the features I need in a text editor:

(Obviously) Vim keybindings.
Syntax highlighting with as much languages supported as possible. Because I will not know what type of file I have to edit!
Very simple configuration overhead. I want to manage just a single file with easy setup. As you might see, I will be trying to avoid some configuration paradigm. So that, when I am in a new setup, I will install VIM (from now, NeoVIM), download and put the file where it is needed and I am all set.
Autocompletion for popular languages.

Why I wanted to switched to NeoVIM?

There are tons of reasons anyone can find in online to switch from VIM to NeoVIM. So in this section, I am not going to restate or go over those. I am pointing just some key points that I must had to say:

Better API and well documented. So if I ever wanted to debug a plugin written for NeoVIM, or wanted to contribute to my daily used text editor, I will have the benefits.
Extensions support any language! (which was not the case in VIM where you’ll have to learn vim script)
Front and Backend is decoupled. This opens possibility to embed NeoVIM anywhere you want!
As I can see in the internet, the world is moving on. So why not me.

Lets switch!

Before starting the switch, here is the old vimrc I use.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36


set viminfo+=n~/.vim/viminfo

" vim-plug
if empty(glob('~/.vim/autoload/plug.vim'))
 silent !curl -fLo ~/.vim/autoload/plug.vim --create-dirs
 \ https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim
 autocmd VimEnter * PlugInstall --sync | source $MYVIMRC
endif

call plug#begin('~/.vim/plugged')

" For sudo edit and others
Plug 'tpope/vim-eunuch'

" For powerline bar
Plug 'itchyny/lightline.vim'

" Linting
Plug 'dense-analysis/ale'

" show +/- in line numbers of a file under git
Plug 'airblade/vim-gitgutter'

" Copy to system clipboard (Requires xsel)
Plug 'christoomey/vim-system-copy'

" Initialize plugin system
call plug#end()

" Required for lightline.vim
set noshowmode
set laststatus=2
set mouse=a

" Other configs goes here
" set number and bla bla

Now, first things first. Install NeoVIM. You can look over their website to know how to install in your operating system. As I am using arch based distro, the command is:

1

yay -S neovim

I found most of the “switch to NeoVIM” blog posts mentioning that it has no problem starting up with the existing vimrc file. But I got a lot of error messages (I solve some of them, but it became cumbersome at a moment, see image below) starting up NeoVIM after installation. As I had time, I preferred to manually move all the configurations from the old vimrc to NeoVIM.

VIM basically uses ~/.vimrc file for its startup. I used to use a trick to move this .vimrc file from ~(home) to ~/.config/vimrc. The trick is to put these 2 lines in any of your init files (in any linux OS). I put them in my ~/.zprofile.

1
2
3


# VIM
export VIMINIT='source $MYVIMRC'
export MYVIMRC='$HOME/.config/vimrc'

This works because, VIM uses VIMINIT environment variable to execute. If VIMINIT is not set in your path, then it looks for ~/.vimrc. This is also same for NeoVIM (I am not 100% sure about this though).

But NeoVIM follows the XDG Base Directory Specification out of the box. So it used ~/.config/nvim/init.vim file for its init configuration. I do not need those 2 lines anymore.

As I am using vim-Plug for plugin management, I need to set it up first. In vimrc, I used the auto-installation code snippet if found here. What it does, is it automatically installs vim-plug if it is not already installed in VIM. Vim-Plug does not have any snippet in their tips page for NeoVIM. After digging a bit, I found this issue where some people suggested similar snippets. After some inspection, I picked the snippet below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


let plug_install = 0
let autoload_plug_path = stdpath('config') . '/autoload/plug.vim'
if !filereadable(autoload_plug_path)
 silent exe '!curl -fL --create-dirs -o ' . autoload_plug_path .
 \ ' https://raw.github.com/junegunn/vim-plug/master/plug.vim'
 execute 'source ' . fnameescape(autoload_plug_path)
 let plug_install = 1
endif
unlet autoload_plug_path

call plug#begin('~/.config/nvim/plugins')
... plugins ...
call plug#end()

if plug_install
 PlugInstall --sync
endif
unlet plug_install

... the rest of the config goes here ...

After this, I copied and pasted all the Plug '...' commands in the right places. The only plugin that did not work for me was vim-eunuch. This plugin adds some UNIX shell commands to use inside vim to do sudo operations.

I searched if there is any alternative plugin for NeoVIM, but unfortunately found this issue. Also another very similar plugin SudoEdit.vim stated the following in their FAQ:

Plugin doesn’t work in neovim, how to make it to work?

Yeah, it is a known problem that neovim broke compatibility here. Since I am not using Neovim, I am not able to fix it.

So, I gave up hope and started using sudo nvim to edit files which I am not the owner of. If you knew any better way or a plugin, I will really appreciate to know 😄.

vim-system-copy is a plugin I use to copy the visual selection from vim to the system clipboard. Vim needs to be compiled with an additional flag to support system clipboard management. On the other hand, NeoVIM supports system clipboard management without any additional compile flags. Inside NeoVIM, there is a variable named clipboard you can use.

So instead of using that plugin, I put this line in my nvim/init.vim file.

1

set clipboard+=unnamedplus

Finally after removing unnecessary lines, the bare-bones config file for my NeoVIM is given below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44


let plug_install = 0
let autoload_plug_path = stdpath('config') . '/autoload/plug.vim'
if !filereadable(autoload_plug_path)
 silent exe '!curl -fL --create-dirs -o ' . autoload_plug_path .
 \ ' https://raw.github.com/junegunn/vim-plug/master/plug.vim'
 execute 'source ' . fnameescape(autoload_plug_path)
 let plug_install = 1
endif
unlet autoload_plug_path

call plug#begin('~/.config/nvim/plugins')
" To edit files in sudo mode
" TODO: :(

" Autocompletion
Plug 'neoclide/coc.nvim', {'branch': 'release'}

" show +/- in line numbers of a file under git
Plug 'airblade/vim-gitgutter'

" For powerline bar
Plug 'itchyny/lightline.vim'

" Comments
Plug 'tpope/vim-commentary'

" And so on
" Plug '...'

call plug#end()

if plug_install
 PlugInstall --sync
endif
unlet plug_install

" Required for lightline.vim
set noshowmode
set laststatus=2
set mouse=a

" Globar configurations
set number
set clipboard+=unnamedplus

Conclusion

As stated in the beginning of this article, I am not a hardcore vim user. I just like the modal editing of vim. So switching to NeoVIM truely does not impact my workflow that much. But everytime I try to edit a file now, I face this issue:

Old habits die hard. Though there is a workaround to use alias vim=nvim, but I prefer to make the alias in my mind.

Thanks for reading this article. Please share anything you do different than my method to switch from vim to neovim.

My 3 Repo went to 2020 Arctic Vault

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 23 Jul 2020 13:22:57 +0600

About GitHub Archive Program

It is a hidden cornerstone of modern civilization, and the shared heritage of all humanity. The mission of the GitHub Archive Program is to preserve open source software for future generations.
– [Github official Archive Program page][1]

A huge archive vault is created in a mine on the Svalbard archipelago, halfway between the Norwegian mainland and the North Pole. It is 0.2 kilometers inside a mountain in the high Arctic.

My 3 Repositories

1. My Linux Dots

I am soon going to be writing a series of posts about my linux distribution and how I use linux in my daily student life. It might be some sort of tutorial for you to follow. There, most of the time I will be pointing to my linux-dots repository. This is the repository where I developed my own linux flavor for my personal usage. Before this repository, I used the repository that is selected for GitHub Archive Program.

I do not know how the repositories are actually selected for Arctic Code Vault, but this repository is unmaintained for about 6 months. I sometimes use this repository as a reference to my new linux dots repository.

2. Yolo Evaluation on Custom Dataset

This is a project created to test the training of several different kinds of data. I used it to test and make presentation of some matrices of local vehicle of Bangladesh (e.g. Rickshaw) detection using transfer learning in Yolo.

The most beautiful thing that I am proud to make is the precision-recall curve of the evaluation. You can have a look at a kernel stored straight in the repository.

Though I have much better public repositories in my GitHub, but this work deserves to be in the Arctic Vault becouse I find it really useful for any machine learning projects.

3. Flow Chart Generator from Hand Drawn Chart and Handwriting Recognition

This is actually not solely my work. A university junior of mine asked me to help develop his third year system development project. So initially I guided him to develop and also forked his repository to contribute.

After the project being praised by our professor, they insisted us to write an article about the work and publish it. So the article was published in IEEE TenSymp international conference.

Repositories in the vault where I contributed

1. WPGTK

WPGTK is a must have program for linux desktop ricers. Basically, ricing is making a computers all softwares pixel perfect and in just the way you want them to work. All your touches to the desktop will be at its most granular level (hence the name ricing!). If you want to get started with linux ricing, I find Dummies Guide to ricing a good start.

WPGTK is a very popular open source software for managing your configuration files and colors in a single place. It also has GUI! (usually, ricing involves only editing configuration files by hand most of the time). There is a lot of popular posts on the one stop subreddit for ricers. Have a look by yourself at /r/unixporn.

I am using this program for my own daily driving linux boxes and am very satisfied with it. The parts with which I was not satisfied are the parts where I contributed. The author of the repository said in own of my PRs is:

It works for me very well, I don’t use xsettingsd or gnome but swaywm. Everything reloads quicker and less hacky. This fixes the issues with the gtk3 reloading very well. Nicely done.
– Jasper (OSS contributor)

2. Zzo theme for Hugo

This is the original Hugo static site theme of this website you are seeing. The work is very wonderful. As I am using this theme for my own website, I wanted to contribute something I can in return (This is what community driven open source software is, isn’t it?).
I have given 4 PRs to this repositories. All 4 were merged. I may also do more contributions alongside developing this personal blog.

How I feel?

Cool:

Okay, my honest opinion is that, it does not matter.
As Arctic Code Vault Contributor badge has been created, many of my friends have showed off their contribution to open source software (Just like me in this page!). But it does not give this badge to those who just contributed to others repositories. For example, take a look at the Linux repositories. It is a huge open source project and in the day I am writing this post, there are 5000+ contributors working in the project. But for this repository, only the repository owner Linus Torvalds will gain a Arctic Contributor Badge.

According to GitHub themselves, they have stated that:

Hi 👋 welcome!
In order for a public repository to be archived in the vault it had to have met the following conditions:

Have commits between 11/13/2019 and 02/02/2020

Have commits from the year before the snapshot (02/03/2019 - 02/02/2020) and at least one star

Have 250 or more stars

If you’ve contributed to repositories that met that criteria at the time of the snapshot, please open a support ticket 12 and we can look into that for you. Be sure to include the specific commits you believe are missing!
– canuckjacq - GitHub Staff

I do not know if he is telling the truth or GitHub has made mistakes to most of the repositories (Because none of my three repositories have met the above conditions).

On the other hand, this badge does not prove you more than just a simple GitHub user (You can say this by looking at the conditions above). So IMHO there is nothing to be excited of that much as some of whom I have seen.

Choosing theme for VSCode

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 29 Jun 2020 00:00:00 +0000

From what I understand is that, visual studio code has primarily two things to maintain.

The icon theme
The color theme

Icon theme for VSCode

Things I usually check while selecting an icon theme for my code editor (Actually file tree to be specific).

The icons should look different: So that I can easily distinguish between them like which one is file and which one is folder.
In my opinion, the icons should be colorful. Because, icons are small. With only grayscale color, the more types of icon you have, the higher your brain will struggle in differentiating between them. Color gives you a new layer of information. So in that scenario, your brain will have less effort in finding the right file.
The icon theme should support as many types of files as possible.

I would choose any icon theme satisfying these categories. As Material Icon Theme is most popular, I just chose it.

N.B. For some person, point 2 can become a bit controversial. Some would say, bringing in color in the sidebar may distract you from the actual code while coding. But it doesn’t bother me as I usually close the file explorer tab while coding 😄 .

Color theme for VSCode

When it comes to color theme, there are literally tons and tons of themes to choose from.

I usually do code for a long time. For example, if you seat for a competetive programming contest, you are going to seat in front of the screen for at least 2 hours straight. In those scenarios, people usually prefer a dark theme because it is good for the eye. So I am going to talk about dark themes only.

As there are soooo many themes for VSCode to choose from, lets shorten the list by taking some popular ones. For example, lets just consider themes from developerdrive website.

In the website, they have showed 5 light and 5 dark themes. I am going to consider only 5 dark themes in the page.

Here are the things I look for while choosing a proper theme for me.

The theme should be as much dark as possible. At the same time, the code should have as much contrast to the background as possible.
I usually don’t like a specific color shade because if you stare at the code for a long time, it might make you hallucinate to see that color on places where they aren’t. For reference, check this colormatters website (specially the See red? part).
All the components highlighted should be highlighted properly. I am saying this explicitly because, I have seen in many themes the following thing.

Let’s for example take this Aurora X theme and open a file in VSCode. Can you see what the problem is?

Of course, the selected and opened file is highlighted, but if you do not have a high contrast ratio monitor, it would be difficult for you to see.

On the other hand, lets take bear theme. As it is less darker than Aurora X theme, the opened file can be identified very easily.

Like the bear theme, One Dark Pro is another theme that satisfies all my requirements. Actually One Dark Pro theme does some things better than bear theme. For example, take a look at the below image of bear theme.

When VSCode is not on focus, the title bar gets some wierd distracting color (I almost thought that this is a bug of the theme). Another problem is, the word match highlighting and markdown header color are very close. This breaks my rule number 3 of choosing a theme.

Finally

At the end, I am still hopping from one theme to another. It is actually to good change and try new themes after one or two weeks. It helps you feel newness to your old project in VSCode.

I am not going to suggest any theme because beauty depends on the eye of the beholder 😉.

Leetcode Problem - Alien Dictionary

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/alien-dictionary/
Difficulty: Hard
Category: Graph

Step 1: Grasping the Problem

This is a popular hard problem where you are provided a group of letters (like dictionaries) and you have to determine the order of the characters the dictionary’s words are sorted by.

In one sentence, we need to sort characters given a group of words. So first we need to detect which characters will come first. Of course the first letter in the first word will come first in lexicographic order. But after that, we will need to keep a relationship between character of who comes before who depending on the order of the words.

So, keeping a relationship between objects tells us that this problem could be solved by graph. And we need to make an order of characters where each character comes first when no other character is above that one in the words list. In other words, a character will be picked only when there are no other characters which depend on it. This is the perfect problem to be solved by topological sorting.

Step 2: Build the graph

To solve the problem with topological sorting, we need a graph first. In this graph, the nodes will be characters and the edges will be the relationship stating “this character must come before the other character”.

While building the graph, we will use a built-in datatype called defaultdict. In python, defaultdict is a dictionary where each non-existent key access will return a default value without raising an exception.

We will use a commonly used built-in generator function named zip. What zip does is it zips two container data structures so that one can easily loop through both of that at the same time. Without zip we would have to do something like this:

1
2
3


for word_ind in range(len(words)-1):
 word1 = words[word_ind]
 word2 = words[word_ind + 1]

But with the help of zip, we can just loop over these without taking extra index variable word_ind:

1

for word1, word2 in zip(words, words[1:]):

So first we create a graph of defaultdict type, loop through the array or words, then loop through the characters of that word, and create an edge for each consecutive words.

1
2
3
4
5
6
7
8


# Build graph
graph = defaultdict(set)
for word1, word2 in zip(words, words[1:]):
 for char1, char2 in zip(word1, word2):
 if char1 != char2:
 if char2 not in graph[char1]:
 graph[char1].add(char2)
 break

While building the graph, an optimization can be done for a corner case. What if multiple words are prefixes of one another. Then there will be no way of sorting them in lexicographic order. We can check then only if the char1 != char2 is always False. And then, the for loop will never hit the break statement. We could use a flag to check if the break is hit or not.

Or, we can use another cool python feature of python named for/else. Yes, else block can be used after a for block in python and it does exactly what is written above. After checking this with else, we can return null string if we cannot sort the characters. So the code will be:

1
2
3
4


# Check that second word isn't a prefix of first word.
else:
 if len(word2) < len(word1):
 return ""

Step 3: Build an `in_degree` dict for topological sorting

According to wikipedia:

For a vertex, the number of head ends adjacent to a vertex is called the indegree of the vertex

Now we need to get to know the in_degree of each node in the graph. Because topological sorting starts (and uses) with the nodes which has in-degrees’ of zero.

First we initialize in_degree with all the characters of words with 0 (this should be done before building the graph so that we can use it while creating the graph)

1
2
3
4
5


# initialize in degree of graph
in_degree = {}
for word in words:
 for char in word:
 in_degree[char] = 0

Finally, we increment a character’s in_degree when we create an edge while building the graph.

1
2
3


if char2 not in graph[char1]:
 graph[char1].add(char2)
 in_degree[char2] += 1 # new line for in_degree

The codes might be looking out of order to you and difficult to understand, so please see the full program given at the end to make sure you understand the order of code snippets used above.

Step 4: Topological Sorting

Topological Sorting is like BFS (Breadth First Search) or DFS (Depth First Search) where instead of starting from a source node, we start from all the nodes that have in_degree[node] == 0. For bfs, we need to use a queue data structure. Python has built-in deque which can be used as a queue with same time and space complexity. Then we push all the nodes that has in_degree[node] == 0 to the queue.

1
2
3
4
5


# Identify vertices that has in_degree = 0
queue = deque()
for char in in_degree:
 if in_degree[char] == 0:
 queue.append(char)

After that, we keep getting nodes from the queue and update all it’s neighboring nodes in_degree. At the same time when popping nodes from the queue, we will append that with out resulting string aliendict.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


# Do the topological sort
aliendict = ""
while queue:
 node = queue.popleft()
 aliendict += node
 # Loop through neighbors set
 for char in graph[node]:
 in_degree[char] -= 1
 if in_degree[char] == 0:
 queue.append(char)

Finally we will need to check if all the characters are inside the aliendict. If they are not inside the aliendict, then we are sure that at some stage, there was a cycle in topological sorting. So there is no single way of sorting the characters given the words. At that time, we will return a null string.

1
2
3
4


if len(aliendict) < len(in_degree):
 return ""

return aliendict

This is the final solution of the program. Topological sorting takes O(|V| + |E|) time and O(|V|) space complexity where V is the list of vertices (characters for word in words in this problem), and E is the list of edges in the graph (|words| * (characters in each word) in this problem). The complete solution is given below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45


from collections import defaultdict, deque

class Solution:
 def alienOrder(self, words: [str]) -> str:
 # initialize in degree of graph
 in_degree = {}
 for word in words:
 for char in word:
 in_degree[char] = 0

 # Build graph
 graph = defaultdict(set)
 for word1, word2 in zip(words, words[1:]):
 for char1, char2 in zip(word1, word2):
 if char1 != char2:
 if char2 not in graph[char1]:
 graph[char1].add(char2)
 in_degree[char2] += 1
 break
 # Check that second word isn't a prefix of first word.
 else:
 if len(word2) < len(word1):
 return ""

 # Identify vertices that has in_degree = 0
 queue = deque()
 for char in in_degree:
 if in_degree[char] == 0:
 queue.append(char)

 # Do the topological sort
 aliendict = ""
 while queue:
 node = queue.popleft()
 aliendict += node
 # Loop through neighbors set
 for char in graph[node]:
 in_degree[char] -= 1
 if in_degree[char] == 0:
 queue.append(char)

 if len(aliendict) < len(in_degree):
 return ""

 return aliendict

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59


class Solution {
public:
 string alienOrder(vector<string> &words) {
 // initialize in degree of graph
 unordered_map<char, int> inDegree;
 for(auto word: words)
 for(auto character: word)
 inDegree[character] = 0;

 // Build graph
 unordered_map<char, vector<char>> graph;
 for(int word_ind = 0; word_ind < words.size() - 1; word_ind++) {
 string word1 = words[word_ind];
 string word2 = words[word_ind + 1];

 bool loopBroken = false;
 for(int char_ind = 0; char_ind < min(word1.length(), word2.length()); char_ind++) {
 char char1 = word1[char_ind];
 char char2 = word2[char_ind];
 if(char1 != char2) {
 if(graph.find(char1) == graph.end()) {
 graph[char1].push_back(char2);
 inDegree[char2]++;
 }
 loopBroken = true;
 break;
 }
 }
 // Check that second word isn't a prefix of first word
 if(!loopBroken && word2.length() < word1.length())
 return "";
 }

 // Identify vertices that has inDegree = 0
 queue<char> q;
 for(auto charDegreePair: inDegree)
 if(charDegreePair.second == 0)
 q.push(charDegreePair.first);

 // Do the topological sort
 string alienDict;
 while(!q.empty()) {
 auto node = q.front();
 q.pop();
 alienDict.push_back(node);
 // Loop through neighbors set
 for(auto character: graph[node]) {
 inDegree[character]--;
 if(inDegree[character] == 0)
 q.push(character);
 }
 }

 if(alienDict.length() < inDegree.size())
 return "";

 return alienDict;
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62


class Solution {
 public String alienOrder(String[] words) {
 // initialize in degree of graph
 Map<Character, Integer> inDegree = new HashMap<Character, Integer>();
 for(String word : words)
 for(Character character : word.toCharArray())
 inDegree.put(character, 0);

 // Build graph
 Map<Character, List<Character>> graph = new HashMap<Character, List<Character>>();
 for(int word_ind = 0; word_ind < words.length - 1; word_ind++) {
 String word1 = words[word_ind];
 String word2 = words[word_ind + 1];

 boolean loopBroken = false;
 for(int char_ind = 0; char_ind < Math.min(word1.length(), word2.length()); char_ind++) {
 Character char1 = word1.charAt(char_ind);
 Character char2 = word2.charAt(char_ind);
 if(char1 != char2) {
 if(!graph.getOrDefault(char1, new ArrayList<Character>()).contains(char2)) {
 List<Character> prevList = graph.getOrDefault(char1, new ArrayList<Character>());
 prevList.add(char2);
 graph.put(char1, prevList);
 inDegree.put(char2, inDegree.get(char2)+1);
 }
 loopBroken = true;
 break;
 }
 }
 // Check that second word isn't a prefix of first word
 if(!loopBroken)
 if(word2.length() < word1.length())
 return "";
 }


 // Identify vertices that has inDegree = 0
 Queue<Character> queue = new LinkedList<>();
 inDegree.forEach((character, charInDegree) -> {
 if(charInDegree == 0)
 queue.add(character);
 });

 // Do the topological sort
 StringBuilder alienDict = new StringBuilder();
 while(!queue.isEmpty()) {
 Character node = queue.remove();
 alienDict.append(node);
 // Loop through neighbors set
 for(Character character :graph.getOrDefault(node, new ArrayList<>())) {
 inDegree.put(character, inDegree.get(character)-1);
 if(inDegree.get(character) == 0)
 queue.add(character);
 }
 }

 if(alienDict.length() < inDegree.size())
 return "";

 return alienDict.toString();
 }
}

Leetcode Problem - All Nodes Distance K in Binary Tree

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/all-nodes-distance-k-in-binary-tree/
Difficulty: Medium
Category: Trees

Step 1: Grasping the Problem

This is purely a graph problem. We are given a binary tree (via a root node), a target node, and an integer K. We need to return a list of nodes at K distance from the target node. For the below example, the target node (marked yellow) is 5 and the answer will be [7, 4, 1] (marked blue), because K = 2.

Below are the cases to handle for solving problem:

The targets children at distance K.
The targets parent node being at distance K.
Other nodes being l+m distant from target: where l is the distance from target to a parent node par, and m is the distance from par to the node. (See the picture above)

If K is even, then you can say that target node is at distance K from itself. We are just assuming that we will always exclude the target node.

Step 2: Build a graph

It can be seen that, there is no escape from traversing from the target node to it’s parent for solving this problem. But it cannot be done in a tree data structure.

So the first step is to convert this tree to a bidirectional unweighted graph. We can traverse the tree and every time we go to a child node, we will assign the parent to the child. This can be done both recursively or iteratively.

Unlike other statically typed programming languages, python has the ability to create class attributes outside of class. We will use this ability for convenience.

1
2
3
4
5
6
7
8


# Time Complexity: O(n)
def dfs(node, parent=None):
 if node: # Check if the node is not None
 node.parent = parent # Create a parent attribute
 dfs(node.left, node)
 dfs(node.right, node)

dfs(root) # Now all nodes in the graph has parent attribute

Step 3: BFS

Now that we have the graph, we can easily apply a BFS (Breadth First Search) from the target to all nodes in the graph, check for the distance and if it is K, log it in a list. Finally, return the list. Again, we can keep the distance saved within the node itself.

If we find a node at distance K, BFS property tells that we have all the nodes at distance K already in the queue. So we can stop BFS and return the queue

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21


# Time Complexity: O(n*n)
target.dist = 0 # distance from itself is zero
queue = [target] # BFS queue
seen = {target} # A set to check if visited

while queue: # run until queue is empty
 if queue[0].dist == K: # Found one of our answer nodes
 # Return the queue
 ans = []
 for node in queue:
 ans.append(node.val)
 return ans

 node = queue.pop(0) # Takes O(len(queue))

 # Go through the neighbors of current node
 for neighbor in (node.left, node.right, node.parent):
 if neighbor and neighbor not in seen:
 neighbor.dist = node.dist + 1
 seen.add(neighbor)
 queue.append(neighbor)

Step 4: Handle corner cases

In this problem, there can be 2 corner cases.

There will be no node at that distance (the graph is very small compared to K).
The graph is empty.

In both cases, we will need to return an empty array.

1

return []

Step 4: Optimize

The complete algorithm currently runs in O(n + n*n) = O(n*n) complexity. Notice that, we are using a list to do a queue’s job in BFS. Python has a built in queue and deque module which have constant access time for all operations. We can use deque for this purpose. Also in some places, we can use list comprehensions which will not effect on complexity, but may have some interpreter optimizations.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13


target.dist = 0
queue = collections.deque([target])
seen = {target}
while queue:
 if queue[0].dist == K:
 return [node.val for node in queue]
 node = queue.popleft()
 for neighbor in (node.left, node.right, node.parent):
 if neighbor and neighbor not in seen:
 neighbor.dist = node.dist + 1
 seen.add(neighbor)
 queue.append(neighbor)
return []

Finally, the time complexity will be O(n) and space complexity is also O(n) where n is the number of nodes in the tree. This is the best because all the n nodes must be visited once to solve the problem.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33


import collections

# Definition for a binary tree node.
class TreeNode:
 def __init__(self, x):
 self.val = x
 self.left = None
 self.right = None

class Solution(object):
 def distanceK(self, root: TreeNode, target: TreeNode, K: int) -> [int]:
 def dfs(node, parent=None):
 if node: # Check if the node is not None
 node.parent = parent # Create a parent attribute
 dfs(node.left, node)
 dfs(node.right, node)
 dfs(root) # Now all nodes in the graph has parent attribute

 target.dist = 0 # distance from itself is zero
 queue = collections.deque([target]) # BFS queue
 seen = {target} # A set to check if visited
 while queue: # run until queue is empty
 if queue[0].dist == K: # Found one of our answer nodes
 return [node.val for node in queue]
 node = queue.popleft()

 # Go through the neighbors of current node
 for neighbor in (node.left, node.right, node.parent):
 if neighbor and neighbor not in seen:
 neighbor.dist = node.dist + 1
 seen.add(neighbor)
 queue.append(neighbor)
 return []

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61


/**
 * Definition for a binary tree node.
 * struct TreeNode {
 * int val;
 * TreeNode *left;
 * TreeNode *right;
 * TreeNode(int x) : val(x), left(NULL), right(NULL) {}
 * };
 */
class Solution {
public:
 vector<int> distanceK(TreeNode* root, TreeNode* target, int K) {
 map<TreeNode*, TreeNode*> parents; // Initial parents for all nodes
 dfs(parents, root, nullptr);

 queue<pair<TreeNode*, int> > q;
 set<TreeNode*> seen;

 q.push({target, 0}); // Distance from itself is zero
 seen.insert(target); // To check if visited
 while(!q.empty()) { // run until queue is empty
 if(q.front().second == K) {
 vector<int> result;
 while(!q.empty()) {
 result.push_back(q.front().first->val);
 q.pop();
 }
 return result;
 }

 // Go through the neighbors of current node
 auto cur = q.front();
 q.pop();
 auto node = cur.first;
 auto dist = cur.second;

 TreeNode* neighbors[] = {
 node->left,
 node->right,
 parents[node]
 };
 for(auto neighbor: neighbors) {
 if(neighbor != nullptr && seen.find(neighbor) == seen.end()) {
 auto new_dist = dist + 1;
 seen.insert(neighbor);
 q.push({neighbor, new_dist});
 }
 }
 }
 return {};
 }

private:
 void dfs(map<TreeNode*, TreeNode*> &parents, TreeNode* node, TreeNode* parent) {
 if(node != nullptr) { // Check if the node is not Node
 parents[node] = parent; // Create a parent
 dfs(parents, node->left, node);
 dfs(parents, node->right, node);
 }
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67


import java.util.*;
import java.lang.*;

/**
 * Definition for a binary tree node.
 * public class TreeNode {
 * int val;
 * TreeNode left;
 * TreeNode right;
 * TreeNode(int x) { val = x; }
 * }
 */
class Pair {
 public TreeNode first;
 public Integer second;
 public Pair(TreeNode first, Integer second) {
 this.first = first;
 this.second = second;
 }
}
class Solution {
 public List<Integer> distanceK(TreeNode root, TreeNode target, int K) {
 Map<TreeNode, TreeNode> parents = new HashMap<>();
 dfs(parents, root, null); // Initial parents for all nodes

 Queue<Pair> queue = new LinkedList<>();
 Set<TreeNode> seen = new HashSet<>();

 queue.add(new Pair(target, 0)); // Distance from itself is zero
 seen.add(target); // To check if visited
 while(!queue.isEmpty()) { // run until queue is empty
 if(queue.peek().second == K) { // Found one of our answer nodes
 List<Integer> result = new ArrayList<>();
 while(!queue.isEmpty())
 result.add(queue.remove().first.val);
 return result;
 }

 // Go through the neighbors of current node
 Pair cur = queue.remove();
 TreeNode node = cur.first;
 int dist = cur.second;

 TreeNode[] neighbors = {
 node.left,
 node.right,
 parents.get(node)
 };
 for(TreeNode neighbor : neighbors) {
 if(neighbor != null && !seen.contains(neighbor)) {
 int new_dist = dist + 1;
 seen.add(neighbor);
 queue.add(new Pair(neighbor, new_dist));
 }
 }
 }
 return new ArrayList<Integer>();
 }

 private void dfs(Map<TreeNode, TreeNode> parents, TreeNode node, TreeNode parent) {
 if(node != null) { // Check if the node is not Node
 parents.put(node, parent); // Create a parent attribute
 dfs(parents, node.left, node);
 dfs(parents, node.right, node);
 }
 }
}

Leetcode Problem - Continuous Median

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/find-median-from-data-stream/
Difficulty: Hard
Category: Arrays

Step 1: Grasping the Problem

This problem is an ad-hoc problem. So there are tons of solutions with similar time and space complexity. Given a stream of input integers, at any time find the median of the integers. According to wiki, the definition of median is:

The median of a finite list of numbers is the “middle” number, when those numbers are listed in order from smallest to greatest.

If there is an odd number of numbers, the middle one is picked. For example, consider the list of numbers

[1, 3, 3, 6, 7, 8, 9]

This list contains seven numbers. The median is the fourth of them, which is 6.

If there is an even number of observations, then there is no single middle value; the median is then usually defined to be the mean of the two middle values.[1][2] For example, in the data set

[1, 2, 3, 4, 5, 6, 8, 9]

The the naive approach will be to sort the integers and pick the middle values (average them if needed). But it will take O(n*logn) for findMedian() function. Can we optimize this more? Notice that the values are given 1 at a time. So while getting the values with addNum(int) function, we can add data-structures specifically to get the medium as fast as possible.

Step 2: Building data structures concept with heaps

A binary heap is an abstract binary tree with the following properties:

A child parent relation will always hold in the whole tree. Most of the time, the relation is lesser than (<) or greater than (>).
It must be a complete binary tree. A complete binary tree is a binary tree in which every level, except possibly the last, is completely filled, and all nodes are as far left as possible.

Below are two representations of a min-heap (a heap where the child-parent relationship is “<”) and a max-heap (a heap where the child-parent relationship is “>”).

1
2
3
4
5
6


 1 10
 / \ / \
 5 3 7 3
 / / \ / \
7 3 5 1 -10
 min-heap max-heap

The above two heaps can be represented in arrays like below:

1
2


min-heap: [1, 5, 3, 7]
max-heap: [10, 7, 3, 3, 5, 1, -10]

The implementation of heaps are very easy to do. But python has built-in heap modules to leverage our coding load. Below is TL;DR of the heaps documentation which is used to solve the problem. To learn more about heapq, please refer to the documentation.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


import heapq

an_array = []

heapq.heappush(an_array, 5)
heapq.heappush(an_array, 3)
heapq.heappush(an_array, 6)
heapq.heappush(an_array, 1)
heapq.heappush(an_array, 9)

print(an_array)
# [1, 3, 6, 5, 9]

print(heapq.heappop(an_array))
# 1
print(heapq.heappop(an_array))
# 3

print(an_array)
# [5, 9, 6] # 5 is top of tree

Step 3: `addNum(int)` implementation

Now that we are familiar with basic operations of heaps, In this problem, we are going to use two heaps to solve. The first one is a min-heap, and the second one is a max-heap. As we just want the middle of the numbers, the max-heap named left will keep the first half of all the numbers so that it has the middle value (or the left value of the middle values if there are even numbers) on top of the heap. And the min-heap named right will hold the second half of all the numbers so that it has the right value of the middle value/s. As there are multiple functions to use in the class, we have to make member variables to use these in all functions of the class.

1
2
3
4


def __init__(self):
 # Initialization
 self.left = [] # A minheap top is left[0]
 self.right = [] # A maxheap top is -right[0]

Python does not have any implementation for max-heaps. But it can be achieved with a very simple trick. Just negate the values before pushing them to the heap!

We will push the new values to the left first, then pop that value back and push it to the right heap. This makes the left and right arrays always heapified.

1
2
3


def addNum(self, num: int) -> None:
 heapq.heappush(self.left, num)
 heapq.heappush(self.right, -heapq.heappop(self.left))

To make the left heap always have the middle value, it needs always to have more or equal values than right heap. This way, it is guaranteed that left will hold the median if the number of elements is odd. But the new value is currently in the right heap (possibly making it bigger than left). So we compare the size of both, and move one value from right to left.

1
2
3



if len(self.left) < len(self.right):
 heapq.heappush(self.left, -heapq.heappop(self.right))

This completes the addNum(int) function implementation.

Step 4: `findMedian()` implementation

Because we have the median at the top of the two heaps, this implementation is pretty straight forward. We will check if we have even or odd values. We can also check if size of left is greater than size of right and return the top of left if not equal (total even numbers) or return the average of two tops if equal (total odd numbers). Remember that the top of the heap is already the first element of the array. And also remember that because the numbers were inserted into the max-heap after negating, you will have to negate it again to get the actual value. So when calculating the medium, we will subtract the top of right instead of adding with top of left.

1
2
3
4


def findMedian(self) -> float:
 if len(self.left) > len(self.right):
 return self.left[0]
 return (self.left[0] - self.right[0]) * 0.5

The complexity of addNum(int) is O(logn) because we are inserting a value in a binary tree. The complexity of findMedian(self) is constant.

The final solution is given below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


import heapq

class MedianFinder:
 def __init__(self):
 # Initialization
 self.left = [] # A minheap top is left[0]
 self.right = [] # A maxheap top is -right[0]


 def addNum(self, num: int) -> None:
 heapq.heappush(self.left, num)
 heapq.heappush(self.right, -heapq.heappop(self.left))

 if len(self.left) < len(self.right):
 heapq.heappush(self.left, -heapq.heappop(self.right))

 def findMedian(self) -> float:
 if len(self.left) > len(self.right):
 return self.left[0]
 return (self.left[0] - self.right[0]) * 0.5

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


import heapq

class MedianFinder:
 def __init__(self):
 # Initialization
 self.left = [] # A minheap top is left[0]
 self.right = [] # A maxheap top is -right[0]


 def addNum(self, num: int) -> None:
 heapq.heappush(self.left, num)
 heapq.heappush(self.right, -heapq.heappop(self.left))

 if len(self.left) < len(self.right):
 heapq.heappush(self.left, -heapq.heappop(self.right))

 def findMedian(self) -> float:
 if len(self.left) > len(self.right):
 return self.left[0]
 return (self.left[0] - self.right[0]) * 0.5

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31


class MedianFinder {
 priority_queue<int> left;
 priority_queue<int, vector<int>, greater<int> > right;
public:
 /** initialize your data structure here. */
 MedianFinder() { }

 void addNum(int num) {
 left.push(num);
 right.push(left.top());
 left.pop();

 if(left.size() < right.size()) {
 left.push(right.top());
 right.pop();
 }
 }

 double findMedian() {
 if(left.size() > right.size())
 return left.top();
 return (left.top() + right.top()) * 0.5;
 }
};

/**
 * Your MedianFinder object will be instantiated and called as such:
 * MedianFinder* obj = new MedianFinder();
 * obj->addNum(num);
 * double param_2 = obj->findMedian();
 */

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35


import java.util.*;
import java.lang.*;

class MedianFinder {
 /** initialize your data structure here. */
 PriorityQueue<Integer> left;
 PriorityQueue<Integer> right;
 public MedianFinder() {
 left = new PriorityQueue<Integer>(); // Java min-heap
 right = new PriorityQueue<Integer>( // Java max-heap
 (Integer val1, Integer val2) -> (- Integer.compare(val1, val2))
 );
 }

 public void addNum(int num) {
 left.add(num);
 right.add(left.remove());

 if(left.size() < right.size())
 left.add(right.remove());
 }

 public double findMedian() {
 if(left.size() > right.size())
 return left.peek();
 return (left.peek() + right.peek()) * 0.5d;
 }
}

/**
 * Your MedianFinder object will be instantiated and called as such:
 * MedianFinder obj = new MedianFinder();
 * obj.addNum(num);
 * double param_2 = obj.findMedian();
 */

Leetcode Problem - Four Sum

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/4sum/
Difficulty: Medium
Category: Arrays

Step 1: Grasping the Problem

Given an array of integers, given an array nums and an integer target, we need to find the list of all unique quadruplets that sum to target. It can be easily perceived that we can do a brute force to check all possible combinations in the array resulting in an O(n^4) time complexity solution.

Of course, there is a better solution than O(n^4). Before solving this problem, it is better to get familiar with the famous 2-sum problem. In 2-sum problem, you are given an array and an integer, and you will need to return the pairs of integers which sum to the given integer. The best solution of 2-sum problem is linear in time complexity. YouTube channel Life at Google has a wonderful video on 2-sum problem to have a great intuition.

This problem can be taken as a generalization of 2-sum problem. We can have two loops over the array with first_index and second_index indices and within the loop, we need to find two number that add up to target - nums[first_index] - nums[second_index]. This becomes a reduction to 2-sum problem indeed.

The algorithm must not return duplicate quadruplets. For this reason, while traversing in the array, we need to make sure that no duplicate values are being looped over. The best way to do this is to sort the array first. We have already 2 outer first_index and second_index loops and a linear algorithm within, the worst case will be at least O(n^3). So sorting the array first will help us find duplicate without hurting the complexity of the algorithm.

A general overview with sample input nums = [1, 0, -1, 0, -2, 2] and target = 0 is visualized below. first_index is named i and second_index is named j to save space.

Step 2: Initialization and sorting the array

As mentioned earlier, to check duplicates, we will sort the array. We will also create a res array where we are going to store the detected quadruplets.

1
2


nums.sort()
res = []

Step 2: first_index loop

The most outer loop will take care of the first element of the resulting quadruplets. We will run this loop from the first element of the array till the last possible element we can include as the first item in resulting quadruplets. As quadruplets has 4 elements, there should be at least 3 elements left in the array for the first_index loop.

1
2


# first loop till third last number
for first_index in range(len(nums)-3):

Step 3: Skip duplicate first element

As the array is now sorted, all equal elements will be neighboring. So after the first iteration of first_index loop, if we again encounter the same element, we need to skip that to avoid duplicates. So we need to check if we are in the first iteration, and if we already have seen this element or not.

1
2


 if first_index > 0 and nums[first_index] == nums[first_index-1]:
 continue

Step 4: second_index loop

The second_index loop is very much similar to the first_index loop. We will start from the next element of first_index, and end leaving 2 elements in the array. In this step, we will also skip the duplicates for the second element of quadruplets.

1
2
3
4


 # Second loop till second last number
 for second_index in range(first_index+1, len(nums)-2):
 if second_index>first_index+1 and nums[second_index] == nums[second_index-1]:
 continue

Step 5: 2-SUM

The first two elements of the quadruplets have been taken care of. Now we just need to solve a 2SUM problem from (second_index+1)th element to the last element. There are many ways to solve this problem. Usually hashset is used to keep track of the seen elements and search in the rest of the array for a complement to sum element. But this is used because the array is not sorted. Using a hashset leads to O(n) space complexity for 2-sum problem. If the array is already sorted, we can use two-pointer shrinking method to solve 2-sum problem.

In two pointer shrinking method, we start with two variables left and right being on two ends of the remaining array. Then we check if we got the desired sum or not. If total value of quadruplet is greater than target, we need a smaller sum, so we move pointer right one index left. If total is smaller than target, we need a bigger sum, so we move pointer left one index right. This process is continued until the pointers overlap each other (or the window is shrank to zero). Finally, if total == target, we have a quadruplet! Simply add it to res. This time, we shrink the window from both sides to keep total as unchanged as possible.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15


# Two sum problem
left = second_index+1
right = len(nums) - 1
while left < right:
 sample = [nums[first_index] + nums[second_index] + nums[left] + nums[right]]

 total = sum(sample)
 if total < target:
 left += 1
 elif total > target:
 right -= 1
 else:
 res.append(sample)
 left += 1
 right -= 1

But this will not be sufficient to solve the problem because we have not yet taken care of duplicates inside the 2-sum window. To do this, while shrinking the window, we see if the value we are getting in left or right is already found or not. This is not necessary if total != target because it will eventually be automatically skipped in the next iteration in while (total will not change, so total != target remains False).

1
2
3
4
5
6
7


 left += 1
 while nums[left] == nums[left-1] and left < right:
 left += 1

 right -= 1
 while nums[right] == nums[right+1] and left < right:
 right -= 1

Step 5: Return result

After all the loops are finished, we can simply return the res variable we have been using to log quadruplets.

1

return res

Step 6: Handle Corner Cases

This problem will work for all natural numbers (will work also for floating points). Also we do not need to check for undersized nums because the first_index loop will never run if len(nums) < 4.

Finally the algorithm takes O(n*n*n) time complexity and O(1) space complexity. This can not be further optimized because there is a proof for 3-sum (similar to 2-sum and 4-sum) problem to take at least O(n^3) time. The complete program is given below.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49


class Solution:
 def fourSum(self, nums: [int], target: int) -> [[int]]:
 # Handle corner case
 if len(nums) < 4:
 return []

 nums.sort()
 res = []

 # first loop till third last number
 for first_index in range(len(nums)-3):
 # Skip duplicates
 if first_index > 0 and nums[first_index] == nums[first_index-1]:
 continue

 # Second loop till second last number
 for second_index in range(first_index+1, len(nums)-2):
 # Skip duplicates
 if second_index>first_index+1 and nums[second_index] == nums[second_index-1]:
 continue

 # Two sum problem
 left = second_index+1
 right = len(nums) - 1
 while left < right:
 sample = [nums[first_index], nums[second_index], nums[left], nums[right]]
 total = sum(sample)

 # move left boundary
 if total < target:
 left += 1
 # move right boundary
 elif total > target:
 right -= 1

 # found match!
 else:
 res.append(sample)

 # move left next until new value found
 left += 1
 while nums[left] == nums[left-1] and left < right:
 left += 1

 # move right next until new value found
 right -= 1
 while nums[right] == nums[right+1] and left < right:
 right -= 1
 return res

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57


class Solution {
public:
 vector<vector<int>> fourSum(vector<int>& nums, int target) {
 // Handle corner case
 if(nums.size() < 4)
 return {};

 sort(nums.begin(), nums.end());
 vector<vector<int> > res;

 // first loop till third last number
 for(int firstIndex = 0; firstIndex < nums.size() - 3; firstIndex++) {
 // Skip duplicates
 if(firstIndex > 0 && nums[firstIndex] == nums[firstIndex-1])
 continue;

 // Second loop till second last number
 for(int secondIndex = firstIndex+1; secondIndex < nums.size() - 2; secondIndex++) {
 // Skip duplicates
 if(secondIndex > firstIndex + 1 && nums[secondIndex] == nums[secondIndex - 1])
 continue;

 int left = secondIndex + 1;
 int right = nums.size() - 1;
 while(left < right) {
 vector<int> sample = {
 nums[firstIndex],
 nums[secondIndex],
 nums[left],
 nums[right]
 };

 int total = accumulate(sample.begin(), sample.end(), 0);

 // move left boundary
 if(total < target)
 left++;
 // move right boundary
 else if(total > target)
 right--;

 // found match!
 else {
 res.push_back(sample);
 left++;
 while(nums[left] == nums[left-1] && left < right)
 left++;
 right--;
 while(nums[right] == nums[right+1] && left < right)
 right--;
 }
 }
 }
 }
 return res;
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60


class Solution {
 public List<List<Integer>> fourSum(int[] nums, int target) {
 // Handle corner case
 if(nums.length < 4)
 return new ArrayList<>();

 Arrays.sort(nums);
 List<List<Integer>> res = new ArrayList<>();

 // first loop till third last number
 for(int firstIndex = 0; firstIndex < nums.length-3; firstIndex++) {
 // Skip duplicates
 if(firstIndex > 0 && nums[firstIndex] == nums[firstIndex - 1])
 continue;

 // Second loop till second last number
 for(int secondIndex = firstIndex+1; secondIndex < nums.length - 2; secondIndex++) {
 // Skip duplicates
 if(secondIndex > firstIndex + 1 && nums[secondIndex] == nums[secondIndex - 1])
 continue;

 int left = secondIndex + 1;
 int right = nums.length - 1;
 while(left < right) {
 List<Integer> sample = new ArrayList<>(
 Arrays.asList(
 nums[firstIndex],
 nums[secondIndex],
 nums[left],
 nums[right]
 )
 );

 int total = 0;
 for(int num: sample)
 total += num;

 // move left boundary
 if(total < target)
 left++;
 // move right boundary
 else if(total > target)
 right--;

 // found match!
 else {
 res.add(sample);
 left++;
 while(nums[left] == nums[left-1] && left < right)
 left++;
 right--;
 while(nums[right] == nums[right+1] && left < right)
 right--;
 }
 }
 }
 }
 return res;
 }
}

Leetcode Problem - Itinerary Reconstruction

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/reconstruct-itinerary/
Difficulty: Hard
Category: Graphs

Step 1: Grasping the Problem

You are given a list of airline tickets in [from, to] format and you will need to construct an itinerary in order starting from a fixed airport "JFK". The key points to mention in this question are:

Lexical ordered itinerary should be constructed if there are multiple itinerary possible.
It is guaranteed that an itinerary with using all tickets is possible.
One must use all the tickets and start from "JFK".

Because there are a lot of visiting and traversal going on, one can surely tell that this is a graph theory problem. So first and foremost thing to do is create a graph from the given tickets.

In the graph, the airports will be nodes and the tickets will be edges. According to the problem stated above, we can say “using all tickets exactly once” means “using all edges exactly once” in the graph. So in this graph problem, one needs to use all edges no matter if you visit the nodes or not. Hearing this, if you are familiar, you will say this is the application of Eulerian Path Construction Algorithm.

There are a lot of algorithms to construct the eulerian path from both directed and undirected graphs e.g. Fleury’s algorithm, Hierholzer’s algorithm etc. Our graph is directed and we will construct the path with a modified version of Hierholzer’s algorithm.

Step 2: Reverse sort the tickets

Because we need the first lexicographic ordered solution, we need to traverse the graph is lexicographic order. For this, we will construct the graph in a way so that the adjacency list’s descendants will be ordered. For this, we will sort the tickets in descending order.

1
2


# To maintain the lexical order
tickets.sort(reverse=True)

Step 2: Build the Graph

The first part of solving a graph theory problem has always been constructing the graph itself. In python it is now-a-days usual to represent a graph with dictionary graph where the keys are sources and values are list of descendants of each edge. This is an adjacency list representation of the graph.

Many use defaultdict to create the graph, but to keep things simple, we will use “default” dict of python instead. Just loop through the edges (tickets in this problem) and append a src dst to the graph dict.

1
2
3
4
5
6


# Build the graph
graph = {}
for src, dst in tickets:
 if src not in graph:
 graph[src] = []
 graph[src].append(dst)

Step 3: Eulerian Path Construction

A graph may or may not have eulerian paths depending on where the edges it has. Hierholzer’s algorithm first checks if the graph has a feasible Eulerian path in it. To check that, it follows the following table.

There are four flavors of Euler path/circuit problem we care about:

X	Eulerian Circuit	Eulerian Path
Undirected Graph	Every vertex has an even degree	Either every vertex has even degree or exactly two vertices have odd degree.
Directed Graph	Every vertex has equal indegree and outdegree	At most one vertex has (outdegree) - (indegree) = 1 and at most one vertex has (indegree) - (outdegree) = 1 and all other vertices have equal in and outdegrees.

This problem guarantees that there must be at least one eulerian path in it. So we do not need to check whether there is one or not.

After detecting the existence of Eulerian path, the algorithm does the two things:

Choose any starting vertex v, and follow a trail of edges from that vertex until returning to v. It is not possible to get stuck at any vertex other than v, because the even degree of all vertices ensures that, when the trail enters another vertex w there must be an unused edge leaving w. The tour formed in this way is a closed tour, but may not cover all the vertices and edges of the initial graph.

As long as there exists a vertex u that belongs to the current tour but that has adjacent edges not part of the tour, start another trail from u, following unused edges until returning to u, and join the tour formed in this way to the previous tour.

Since we assume the original graph is connected, repeating the previous step will exhaust all edges of the graph.

So we can use a list named route to keep track of “joining the tour formed in this way”. We can also create and call a recursive function to do “repeating the previous step” to “exhaust all edges” and get the final route in backward.

A demonstration of the algorithm is given below. Note that, this graph does not have a valid itinerary to use all tickets, still the algorithm takes as much tickets as possible and build an itinerary.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


# Eulerian path
route = []
def visit(airport):
 # DFS-like visiting all node
 while airport in graph and graph[airport]:
 dst = graph[airport].pop()
 visit(dst)
 # push the first non-visited node
 route.append(airport)
visit('JFK')

Step 4: Reverse and Return `route`

Finally, we have route with the eulerian path in backward direction. So we will reverse the path and return it as a result.

1
2


# Return in reverse
return reversed(route)

The time complexity of this problem is O(ELogE) where E is the number of edges (tickets), and V is the number of nodes (airports). The space complexity is order of O(E) because we are just storing the graph with a dict. The small but efficient full code is given below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25


class Solution:
 def findItinerary(self, tickets: [[str]]) -> [str]:
 # To maintain the lexical order
 tickets.sort(reverse=True)

 # Build the graph
 graph = {}
 for src, dst in tickets:
 if src not in graph:
 graph[src] = []
 graph[src].append(dst)

 # Eulerian path
 route = []
 def visit(airport):
 # DFS-like visiting all node
 while airport in graph and graph[airport]:
 dst = graph[airport].pop()
 visit(dst)
 # push the first non-visited node
 route.append(airport)
 visit('JFK')

 # Return in reverse
 return reversed(route)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38


class Solution {
 unordered_map<string, vector<string> > graph;
public:
 vector<string> findItinerary(vector<vector<string>>& tickets) {
 // To maintain the lexical order
 sort(tickets.begin(), tickets.end(), [] (vector<string> &first, vector<string> &second) -> bool {
 bool tmp = lexicographical_compare(first[0].begin(), first[0].end(), second[0].begin(), second[0].end());
 if(first[0] == second[0])
 return lexicographical_compare(first[1].begin(), first[1].end(), second[1].begin(), second[1].end());
 return tmp;
 });
 reverse(tickets.begin(), tickets.end());

 // Build the graph
 for(auto ticket: tickets)
 graph[ticket[0]].push_back(ticket[1]);

 // Eulerian path
 vector<string> route;
 visit("JFK", route);

 // Return in reverse
 reverse(route.begin(), route.end());
 return route;
 }

private:
 void visit(string airport, vector<string> &route) {
 // DFS-like visiting all node
 while(graph.find(airport) != graph.end() && graph[airport].size() != 0) {
 auto dst = graph[airport].back();
 graph[airport].pop_back();
 visit(dst, route);
 }
 // push the first non-visited node
 route.push_back(airport);
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29


class Solution {
 Map<String, PriorityQueue<String>> graph;

 public List<String> findItinerary(List<List<String>> tickets) {
 // Unlike python, where we manually sorted tickets.
 // We will use map with PriorityQueue as value which is faster

 // Build the graph
 graph = new HashMap<>();
 for (List<String> ticket : tickets) {
 graph.putIfAbsent(ticket.get(0), new PriorityQueue<>());
 graph.get(ticket.get(0)).add(ticket.get(1));
 }

 // Eulerian path
 LinkedList<String> route = new LinkedList<>();
 visit("JFK", route);

 return route;
 }

 private void visit(String airport, LinkedList<String> route) {
 PriorityQueue<String> priorityQueue = graph.get(airport);
 // DFS-like visiting all node
 while(priorityQueue != null && !priorityQueue.isEmpty())
 visit(priorityQueue.poll(), route);
 route.addFirst(airport);
 }
}

Leetcode Problem - Jump Game II

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/jump-game-ii/
Difficulty: Hard
Category: Dynamic Programming

Step 1: Grasping the Problem

Many people classify this popular “Minimum number of jumps” problem into a dynamic programming problem. In spite of it being truely a dynamic programming problem, it can be solved without noticing any dynamic programming behavior in it. The problem will give you a non-negative integer array nums, where each element defines the maximum length of jump one can make from that index. Starting from index 0, you need to determine the minimum number of jumps required to reach the end of the array.

Below are some important inference one might miss from this problem.

Although the problem does not say anything whether you can jump backward or not, it is not necessary because each element of the array represents maximum jump length. That means of course, you can make shorter jumps in order to get the best result.
From an element, the farthest point we can go will never decrease for the forthcoming elements. So if we are at ith stair and the farthest stair we could go is farthest_possible, farthest_possible will always increase for i+k where k is |nums|-i.
We do not need the last value of nums. Because we will never jump from the last index (it is the destination).

Step 2: Keep track of farthest stair possible

We can start iterating through nums and keep track of the farthest stair we can go from any previous index. For index i, it will be max(k + nums[k]) where k = 0:i. We will also initialize a variable names total_jumps to store the result. Notice how we are leaving the last index of the array in the loop.

1
2
3
4
5
6


total_jumps = 0
current_end = 0 # We will use this on step 3
farthest_possible = 0 # Or any negative value
for jump_ind, jump_len in enumerate(nums[:-1]):
 # Update farthest index I can reach
 farthest_possible = max(farthest_possible, jump_ind + jump_len)

Step 3: Keep track of last jump end

When we reach at a stair which is farthest from the last jump we made, we need to make a new jump. And this jump is not necessarily needed to make from the current index. We can make a jump from any stair before (preferably from that stair which will send us to the farthest_possible).

But as soon as we make a jump, we need to keep track of the farthest jump during that time (let this stored variable be current_end). Next time at each index, we will check if we have reached the end of the last jump or not.

1
2
3
4


# We have to use a new jump from any previous stair and reach farthest
if jump_ind == current_end:
 total_jumps += 1
 current_end = farthest_possible

The visualization of how farthest_possible, current_end and total_jumps work is demonstrated below.

Step 4: Return the total jumps

After the iteration ends, total_jumps will hold the total jumps needed to reach the last index.

1

return total_jumps

We are assuming that it is guaranteed to reach the end of the array. But it is pretty easy to handle, just check if farthest_possible is less than jump_ind.

This algorithm runs only a single loop, so it takes O(n) time and O(1) space complexity. The complete program is given below.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14


class Solution:
 def minJump(self, nums: [int]) -> int:
 total_jumps = 0
 current_end = 0
 farthest_possible = 0

 for jump_ind, jump_len in enumerate(nums[:-1]):
 # Update farthest index I can reach
 farthest_possible = max(farthest_possible, jump_ind + jump_len)
 # We have to use a new jump from any previous stair and reach farthest
 if jump_ind == current_end:
 total_jumps += 1
 current_end = farthest_possible
 return total_jumps

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17


class Solution {
public:
 int jump(vector<int>& nums) {
 int totalJumps = 0, currentEnd = 0, farthestPossible = 0;
 for(int jumpI = 0; jumpI < nums.size() - 1; ++jumpI) {
 int jumpLen = nums[jumpI];
 // Update farthest index I can reach
 farthestPossible = max(farthestPossible, jumpI + jumpLen);
 // We have to use a new jump from any previous stair and reach farthest
 if(jumpI == currentEnd) {
 totalJumps++;
 currentEnd = farthestPossible;
 }
 }
 return totalJumps;
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16


class Solution {
 public int minJump(int[] nums) {
 int totalJumps = 0, currentEnd = 0, farthestPossible = 0;
 for(int jumpI = 0; jumpI < nums.length - 1; jumpI++) {
 int jumpLen = nums[jumpI];
 // Update farthest index I can reach
 farthestPossible = Math.max(farthestPossible, jumpI + jumpLen);
 // We have to use a new jump from any previous stair and reach farthest
 if(jumpI == currentEnd) {
 totalJumps++;
 currentEnd = farthestPossible;
 }
 }
 return totalJumps;
 }
}

Leetcode Problem - Longest Substring Without Duplication

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/longest-substring-without-repeating-characters/
Difficulty: Hard
Category: Strings

Step 1: Grasping the Problem

We need to get the length of the maximum substring from a given string where the substring must not have any duplicate characters. We do not need to return the substring. So we do not need to care about the order of the substrings.

As we do not need the string, we just need to take care of the frequencies of the largest string. In a more “targeting towards the solution” way, we need to keep the characters of largest string so far stored so that when we try to make the string larger by concatenating more characters, we can make sure that it is not already in the stored characters. Hashset is a perfect data-structure for the following use. Python has built in set for this.

Also, a substring is a sequential subset of the given string string. And we need to expand this sequential subset as large as possible. This tells us that a sliding window maximization approach is best to solve this problem.

In a sliding window maximization problem, we start from the first element with window size one. Then we try to expand the window by one-by-one element until we can’t. Then we shrink the window until again we become able to increase it. Everytime we increase, we could track both the largest substring and it’s length. Below is a small demonstration of what a sliding window approach would look like.

Step 2: Sliding Window Initialization

A sliding window needs four things to operate.

The left boundary of the window (left for this solution).
The right boundary of the window (right for this solution).
The target variable (longest for this solution). This variable stores the target we want to get from the window. It can be many variables or states like the starting and ending of the substring. But in this solution, we only need the length of the largest substring.
Any intermediate variables to assert the conditions of window expanding/shrinking ability. In this problem, it is current_chars which stores the characters of the largest string.

So the initialization will look something like this:

1
2
3


longest = 0 # Answer
left, right = 0, 0 # Window bounds
current_chars = set() # Chars within current window

Step 3: Loop for the window

The sliding window will run until we have reached the end of the string. But do we need the left variable to reach the end too? No, because we need only the largest value. Moving left towards end will only shrink the window making longest smaller that it is now.

So looping until right has reached end will suffice.

1

while right < len(string):

Step 4: Expand the window

We can only expand the window when the new character is not in current_chars list. After checking if we can add the character, we need to add that character to current_chars to check for new characters in the future.

We also need to increase longest if it becomes larger than the previous one. And finally we need to expand the window boundary to one character right.

1
2
3
4
5


# Expand window
if string[right] not in current_chars:
 current_chars.add(string[right])
 right += 1
 longest = max(longest, right - left)

Step 5: Shrink the window

If we cannot expand the window, we will need to shrink it. We will shrink it until the new character is removed from current_chars. This can be taken care by the while loop we created earlier. And after removing the leftmost character of the window, we will move the left boundary of the window one character right.

1
2
3
4


# Shrink window
else:
 current_chars.remove(string[left])
 left += 1

Step 6: Return `longest`

After the while loop ends, we will surely get the longest substrings length. We will return it after the while loop.

1

return longest

Every sliding window problem takes O(n) time itself, where n is the length of the string. Inside the loop, we are making a look-up in a hashset and incrementing and decrementing variable. All these take O(1) time. So the time complexity of the algorithm is O(n) and space complexity is also O(n) in the worst case. The program is given below.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16


class Solution:
 def lengthOfLongestSubstring(self, string: str) -> int:
 longest = 0 # Answer
 left, right = 0, 0 # Window bounds
 current_chars = set() # Chars within current window
 while right < len(string):
 # Expand window
 if string[right] not in current_chars:
 current_chars.add(string[right])
 right += 1
 longest = max(longest, right - left)
 # Shrink window
 else:
 current_chars.remove(string[left])
 left += 1
 return longest

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22


class Solution {
public:
 int lengthOfLongestSubstring(string str) {
 int longest = 0; // Answer
 int left = 0, right = 0; // Window bounds

 // Chars within current window
 set<char> currentChars;
 while(right < str.length()) {
 // Expant window
 if(currentChars.find(str[right]) == currentChars.end()) {
 currentChars.insert(str[right++]);
 longest = max(longest, right - left);
 }
 // Shrink window
 else {
 currentChars.erase(str[left++]);
 }
 }
 return longest;
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23


class Solution {
 public int lengthOfLongestSubstring(String string) {
 int longest = 0; // Answer
 int left = 0, right = 0; // Window bounds

 // Chars within current window
 Set<Character> currentChars = new HashSet<>();
 while(right < string.length()) {
 // Expand window
 if(!currentChars.contains(string.charAt(right))) {
 currentChars.add(string.charAt(right));
 right++;
 longest = Math.max(longest, right - left);
 }
 // Shrink window
 else {
 currentChars.remove(string.charAt(left));
 left++;
 }
 }
 return longest;
 }
}

Leetcode Problem - Lowest Common Ancestor

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/lowest-common-ancestor-of-a-binary-tree/
Difficulty: Medium
Category: Trees

Step 1: Grasping the problem

This is a tree problem where a tree (by root) and two nodes (node1 and node2) within the tree is given. We will need to return LCA of node1 and node2.

Lowest Common Ancestor (LCA) of two nodes is defined as the node which is ancestor of both node1 and node2 and is at the lowest level of tree as possible. For example, in the below tree, node1 = 6 and node2 = 4. The LCA node cannot be 2 because node 2 is not an ancestor of node1 On the other hand, node 3 cannot be LCA node because there is another node (node 5) which is ancestor of both node1 and node2 and is in deeper level than 3. So the answer must be node 5.

Step 2: Add parent attribute

Currently only the root of the tree is given. That means we can only go from root to any child. But to solve this problem, we will need to be able to traverse from a child node to it’s parent. So the first task is to add a new data parent for each node so that we can easily traverse both down and up in the tree.

Unlike other statically typed programming languages, python has the powerful ability to create class attributes from outside of a class. We will use this to create the parent attribute right in each nodes.

To add an attribute to each node we need to traverse through all the nodes once. Any tree traversal algorithm can be used here. We are using simple DFS here. Go ahead and change the stack to a queue to do a BFS traversal.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


root.parent = None
# Start a DFS and assign parent to each node
stack = {root}
while stack:
 node = stack.pop()
 if node.left:
 node.left.parent = node
 stack.append(node.left)
 if node.right:
 node.right.parent = node
 stack.append(node.right)

Step 3: Store the path of `node1`

Now we need to store the path of node node1 and then look where the path conflicts with node node2’s path. node1 and node2 are interchangeable in this case so you can store the path of node2 and look up which node in node1 path conflicts with it.

As we already have the node node1 in hand, with the help of parent attribute we just created, we will traverse up in the tree till root and store all the nodes in the path.

This stored list of nodes will only be used for lookup of a node. So instead of using list as the data structure, we can use a hashset implementation set which is similar to unordered_set<int> in C++ and efficient in key lookups.

1
2
3
4
5


# Trace the path of node node1
node1_path = set()
while node1:
 node1_path.append(node1)
 node1 = node1.parent

Step 4: Search for first existent node

Now it is needed to do the almost same thing with node node2. Except for storing the nodes of the path, we will loop up the node in node1_path. The first node we find will be our answer!

1
2
3
4
5


# Search for first node of node2's parent in node1_path
while node2:
 if node2 in node1_path:
 return node2
 node2 = node2.parent

This is it. A good practice though is to return a value at the end of the function (just in case one cannot find and claim the code to be broken).

1

return None # A good practice

So the overall time complexity of the function is O(n) because there is nothing but traversal we are doing. The space complexity is also O(n) because we are storing the path of one node (node1 or node2). There are algorithms where you can make the space complexity to O(1) in the cost of more time. Below is the complete code of the solution.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35


# Definition for a binary tree node.
class TreeNode:
 def __init__(self, x):
 self.val = x
 self.left = None
 self.right = None

class Solution:
 def lowestCommonAncestor(self, root: TreeNode, node1: TreeNode, node2: TreeNode) -> TreeNode:
 root.parent = None

 # Start a DFS and assign parent to each node
 stack = [root]
 while stack:
 node = stack.pop()
 if node.left:
 node.left.parent = node
 stack.append(node.left)
 if node.right:
 node.right.parent = node
 stack.append(node.right)

 # Trace the path of node node1
 node1_path = set()
 while node1:
 node1_path.add(node1)
 node1 = node1.parent

 # Search for first node of node2's parent in p_path
 while node2:
 if node2 in node1_path:
 return node2
 node2 = node2.parent

 return None # A good practice

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46


/**
 * Definition for a binary tree node.
 * struct TreeNode {
 * int val;
 * TreeNode *left;
 * TreeNode *right;
 * TreeNode(int x) : val(x), left(NULL), right(NULL) {}
 * };
 */
class Solution {
public:
 TreeNode* lowestCommonAncestor(TreeNode* root, TreeNode* node1, TreeNode* node2) {
 // Start a DFS and assign to each node
 stack<TreeNode*> st;
 st.push(root);
 map<TreeNode*, TreeNode*> parents;
 while(!st.empty()) {
 auto node = st.top();
 st.pop();
 if(node->left != nullptr) {
 parents[node->left] = node;
 st.push(node->left);
 }
 if(node->right != nullptr) {
 parents[node->right] = node;
 st.push(node->right);
 }
 }

 // Trace the path of node node1
 set<TreeNode*> node1Path;
 while(node1 != nullptr) {
 node1Path.insert(node1);
 node1 = parents.find(node1) != parents.end() ? parents[node1] : nullptr;
 }

 // Search for first node of node2's parent in node1Path
 while(node2 != nullptr) {
 if(node1Path.find(node2) != node1Path.end()) {
 return node2;
 }
 node2 = parents.find(node2) != parents.end() ? parents[node2] : nullptr;
 }
 return nullptr; // Must return in C++
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45


/**
 * Definition for a binary tree node.
 * public class TreeNode {
 * int val;
 * TreeNode left;
 * TreeNode right;
 * TreeNode(int x) { val = x; }
 * }
 */

class Solution {
 public TreeNode lowestCommonAncestor(TreeNode root, TreeNode node1, TreeNode node2) {

 // Start a DFS and assign parent to each node
 Stack<TreeNode> stack = new Stack<>();
 stack.push(root);
 Map<TreeNode, TreeNode> parents = new HashMap<>();
 while(!stack.isEmpty()) {
 TreeNode node = stack.pop();
 if(node.left != null) {
 parents.put(node.left, node);
 stack.push(node.left);
 }
 if(node.right != null) {
 parents.put(node.right, node);
 stack.push(node.right);
 }
 }

 // Trace the path of node node1
 Set<TreeNode> node1Path = new HashSet<TreeNode>();
 while(node1 != null) {
 node1Path.add(node1);
 node1 = parents.getOrDefault(node1, null);
 }

 // Search for first node of node2's parent in node1_path
 while(node2 != null) {
 if(node1Path.contains(node2))
 return node2;
 node2 = parents.getOrDefault(node2, null);
 }
 return null; // Must return in Java
 }
}

Leetcode Problem - Maximum Profit with K Transactions

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/best-time-to-buy-and-sell-stock-iv/
Difficulty: Hard
Category: Dynamic Programming

Step 1: Grasping the Problem

This problem gives an array of integer prices denoting stock prices each day and an integer k denoting the maximum number of transactions that can be made. We need to return the maximum profit possible for that stock. Some key points to notice in this problem is:

There cannot be multiple simultaneous transactions. So there will not be any overlaps in the transactions.
You can have at most |prices|/2 transactions where |prices| is the number of days.
If k > |prices|*2, then it can be said that unlimited transactions can be made. Because you can make at most |prices|/2 transactions as stated in the previous point.

This tells us that we may have to handle the scenario where k>|price|*2 and the normal cases separately.

The problem statement tells that we can do at most k transactions, that means that we can have less than k transactions. Notice that you can break any transaction to multiple transactions if that takes more that 2 days. Below is an example where prices = [2, 4, 5, 7]. To get a profit of 7 - 2 = 5, you can make two transactions: first one buying at day 1 and selling at day 3 (profit = 5-2 = 3), and the second one buying at day 3 and selling at day 4 (profit = 7-5 = 2). This gives a profit of 3 + 2 = 5. And you can also do only one transaction buying at day 1 and selling at day 4.

Now suppose k=1. In that case, you’ll need to check for only one transaction in the whole prices and that would be the result. Then think of k=2. We can do the following things at each day in that scenario:

If we are not holding any transaction, then we can buy a new transaction. Buying means that we are subtracting the price of that day from our total amount. By this process, we can make at most k transactions.
If we are holding a transaction, we can sell a transaction: selling ith transaction means, the total amount becoming the the last time we gave money to buy a transaction plus the price of today’s stock.

So, we can track all the states of the transactions and prices by tracking our total amount for each buy and sell, making new transactions by adding or subtracting todays price from the last transaction.

This gives us intuition about the problem being a DP (Dynamic programming) problem. Here, we will use two arrays to store the maximum amount possible at each transaction. We do not need to track this based on days, because only the transactions make changes to those states.

buy[t] : Amount of money while buying for the t'th transaction.
sell[t]: Amount of money while selling for the t'th transaction.

Step 2: Create a DP table

DP (Dynamic Programming) is a programming paradigm where there can be multiple dependent subproblems. There are two different ways to approach a DP problem: Recursively and iteratively. We will be showing the iterative approach as it can be elaborated more with data. For iterative Dynamic Programming, you will need to fill up a table which is known as DP table. As our problem is only dependent on the last transaction from the current transaction, and there are at most k transactions possible, we will use a DP table of k*2 size, where the first row of the table is the track of best buy transaction amounts, and the second row is track of best sell transaction amounts. Now the initial values of the buy row will be a minimum value, so that we can compute them with a negative value. We will need this in future. On the other hand, sell row will be initialized to zero because, without making any transactions you’ll get total amount of zero.

transaction	1	2	…	k
buy	-∞	-∞	-∞	-∞
sell	0	0	0	0

This can be done either by creating a 2D array, or buy creating just two variable buys and sells. Multiplying an array with an integer in python will result in concatenating the array with itself. We can use this to initialize the value.

1
2
3


# Build the table
buys = [float('-inf')] * k
sells = [0 ] * k

Step 3: Fill up DP table

As we have defined how we are going to store the states needed for the problem, we will need a state transition function. We will do this by first iterating through the prices stock prices list, and then iterating through the number of transactions we can make at most.

1
2


for price in prices:
 for transaction in range(k):

Initially when we do not have any transactions done, the price will be -price where price is the price of the first day. This will be negative so that when we sell this stock, the price will be new_buy = -price(at day 1) + price(at day of selling). Of course the selling price will have to be greater than the buying price so that we can make a profit. We will use a new variable new_buy to store this.

1
2


if transaction == 0:
 new_buy = -price

Now what will be the amount when it is not the first transaction? Then we will already be having a profit in our hand. And this value will be in the sells row’s last transaction column (sells[transaction - 1]). Then we will subtract todays price from that value.

1
2


else:
 new_buy = sells[transaction-1] - price

Now that we have got the value we need for what happens if we buy a transaction or not, we can update the buys row. We will only buy if it is beneficial than not buying the transaction.

1
2


if new_buy > buys[transaction]:
 buys[transaction] = new_buy

After updating the buy for current transaction, We will see if we can sell beneficially today. We will sell today only if it is profitable, meaning adding todays stock price to the last buys[transaction] becomes greater thatn sells[transaction].

1
2


if buys[transaction] + price > sells[transaction]:
 sells[transaction] = buys[transaction] + price

In most of the languages, the above two updating can be done in one line with the help of built-in max function.

1
2


buys[transaction] = max(buys[transaction], new_buy)
sells[transaction] = max(sells[transaction], buys[transaction] + price)

After filling up the table, for example with prices = [5, 11, 3, 50, 60, 90] and k = 2, the table will look like below.

price	transaction	buys	sells
5	0	[-5, -inf]	[0, 0]
5	1	[-5, -5]	[0, 0]
11	0	[-5, -5]	[6, 0]
11	1	[-5, -5]	[6, 6]
…	…	…	…
60	0	[-3, 3]	[57, 53]
60	1	[-3, 3]	[57, 63]
90	0	[-3, 3]	[87, 63]
90	1	[-3, 3]	[87, 93]

Step 4: Handling corner cases

There can be two corner cases for this problem.

The typical corner case: What if you have no stock prices? |prices| = 0. What happens if you can make 0 transactions? k = 0. In both of these scenarios, we can tell that we will have a maximum of 0 profit. So we return 0.

1
2
3


# Corner cases 1: Usual
if not prices or k == 0:
 return 0

Unlimited transactions: As we have already stated in Step 1, you can have at most |prices|*2 transactions? So what if k > |prices|*2? Then you can easily calculate assuming you have unlimited transactions.

1
2
3
4
5
6
7
8


# Corner case 2: unlimited transactions
if k >= len(prices)/2:
 profit = 0
 for price_index in range(len(prices)-1):
 new_profit = prices[price_index+1] - prices[price_index]
 if new_profit > 0:
 profit += new_profit
 return profit

Note that these cases should be handled before going for the actual DP generation. Because there is no need to have a DP table if you already know you can have unlimited transactions.

Step 5: Returning the result

At this point, we have the DP table filled up properly. So what we can do is return our profit at k‘th transaction. We know that our profit is saved in sells row. So returning the k‘th column of it will be sufficient.

1

return sells[k-1]

The overall program is given below. As we had to loop through the prices list and number of transactions (k), the time complexity will be O(|prices| * k). The space complexity will also be O(|prices| * k) for the DP table.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28


class Solution:
 def maxProfit(self, k: int, prices: [int]) -> int:
 # Corner cases 1: Usual
 if not prices or k == 0:
 return 0

 # Corner case 2: unlimited transactions
 if k >= len(prices)/2:
 profit = 0
 for price_index in range(len(prices)-1):
 new_profit = prices[price_index+1] - prices[price_index]
 if new_profit > 0:
 profit += new_profit
 return profit

 # Build the table
 buys = [float('-inf')]*k
 sells = [0]*k
 for price in prices:
 for transaction in range(k):
 if transaction == 0:
 new_buy = -price
 else:
 new_buy = sells[transaction-1] - price

 buys[transaction] = max(buys[transaction], new_buy)
 sells[transaction] = max(sells[transaction], buys[transaction] + price)
 return sells[k-1]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33


class Solution {
public:
 int maxProfit(int k, vector<int>& prices) {
 // Corner cases 1: Usual
 if(prices.size() == 0 || k == 0)
 return 0;

 // Corner case 2: unlimited transactions
 if(k >= prices.size()/2) {
 int profit = 0;
 for(int priceIndex = 0; priceIndex < prices.size()-1; priceIndex++) {
 int newProfit = prices[priceIndex + 1] - prices[priceIndex];
 if(newProfit > 0)
 profit += newProfit;
 }
 return profit;
 }

 // Build the table
 vector<int> buys(k, INT_MIN);
 vector<int> sells(k, 0);
 for(int price: prices) {
 for(int transaction = 0; transaction < k; transaction++) {
 int new_buy = -price;
 if(transaction != 0)
 new_buy = sells[transaction - 1] - price;
 buys[transaction] = max(buys[transaction], new_buy);
 sells[transaction] = max(sells[transaction], buys[transaction] + price);
 }
 }
 return sells[k-1];
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39


class Solution {
 public int maxProfit(int k, int[] prices) {
 // Corner cases 1: Usual
 if(prices.length == 0 || k == 0)
 return 0;

 // Corner case 2: unlimited transactions
 if(k >= prices.length/2) {
 int profit = 0;
 for(int priceIndex = 0; priceIndex < prices.length - 1; priceIndex++) {
 int newProfit = prices[priceIndex+1] - prices[priceIndex];
 if(newProfit > 0)
 profit += newProfit;
 }
 return profit;
 }

 // Build the table
 int[] buys = new int[k];
 for(int buyI = 0; buyI < k; buyI++)
 buys[buyI] = Integer.MIN_VALUE;

 int[] sells = new int[k];
 for(int sellI = 0; sellI < k; sellI++)
 sells[sellI] = 0;

 for(int price: prices) {
 for(int transaction = 0; transaction < k; transaction++) {
 int new_buy = -price;
 if(transaction != 0)
 new_buy = sells[transaction - 1] - price;

 buys[transaction] = Math.max(buys[transaction], new_buy);
 sells[transaction] = Math.max(sells[transaction], buys[transaction] + price);
 }
 }
 return sells[k-1];
 }
}

Leetcode Problem - Minimum Window Substring

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/minimum-window-substring/
Difficulty: Hard
Category: Strings

Step 1: Grasping the Problem

A strings text and characters is given. We need to return the smallest substring of text which contains all the characters in characters.

The problem would have been easier if it said all the characters is the string characters will be unique. Then we could just manage a set and push characters from text until we get all the characters. But as the characters might not be unique, instead of set, we will need a hashtable/counter to keep track of the frequencies of each characters we encounter in string text.

A substring is a sequential subset of the given string text. And we need to minimize this sequential subset to as small as possible. This tells us that a sliding window minimization approach is best to solve this problem.

We will try to keep the sliding window as small as possible. For this, first we need to start the window with 0 length. We will have to option other than increasing the window size until a valid substring is found. Whenever substring is found, we will keep it as a potential result and try shrinking the window for smaller substring. When we cannot shrink the window, we will start increasing it again until the right bound of the window reaches the end.

Below is a visual demonstration of the process. Here, text = "ADCBCAAB" and characters = "BCAC".

Step 2: Sliding Window Initialization

A sliding window needs four things to operate.

The left boundary of the window (left for this solution).
The right boundary of the window (right for this solution).
The target variable (best_text for this solution). This variable stores the target we want to get from the window. It can be of many variables or states like the starting and ending of the substring (which will be more efficient). But for simplicity, in this solution, we will take the substring directly.
Any intermediate variables to assert the conditions of window expanding/shrinking ability. In this problem, we need two python collections.Counter objects to keep track if we have all the characters we need.

Initially, we want the target variable to be the worst or unfeasible. So that it will be overwritten by any existent solution in the window expanding/shrinking loop. The initialization will look something like this:

1
2
3
4
5
6
7


chars_given = Counter(characters) # count characters
chars_taken = Counter() # count remaining characters

best_text = "-" * (len(text) + 1) # bigger than text

left = 0 # Window left
# right window pointer can be moved into for loops dummy variable

Step 3: Expand the window

The sliding window will run until we have reached the end of the string. But do we need the left variable to reach the end too? Yes, because we are going to need the smallest value. But we do not need that condition to check in the outer loop, we will take care in the shrinking loop.

So looping until right has reached end will suffice. And at each time window is shrank, we will get a character char. We will take it in our `chars

1
2


for right, char in enumerate(text):
 chars_taken[char] += 1 # Expand window

Step 4: Shrinking ability check

We can only expand the window when we have all the characters we need in our chars_taken counter. One way to check this is looping through all the elements in chars_given and check if that character in chars_taken is greater. Something like below:

1
2
3
4
5
6


have_all = True
for char in chars_given:
 if chars_given[char] >= chars_taken[char]:
 have_all = False
if have_all:
 # We have all the characters, so we can shrink

But python has a more elegant & operator with better time complexity to handle this. chars_given & chars_taken returns intersection of the two counters. We need to check if the intersection is equal to chars_given.

1
2


# While window can be shrunk (operator & is intersection)
while chars_taken & chars_given == chars_given:

Step 5: Get Minimum Substring

We will need to update best_text everytime when we change the window size. This is for the following reasons.

When expanding the window: It would not be necessary at all if we had guaranteed substring. But if it wasn’t guaranteed, best_text needs to be updated if there is only one string matching our criteria.
When shrinking the window: We are going to need the smallest substring matching the criteria, so each time the window shrinks, there is a possibility of getting a smaller substring, so we need to update to the smallest.

We will check the window size (right - left) with the length of best_text then update best_text by slicing if needed.

1
2
3


# Update best_text
if right - left < len(best_text):
 best_text = text[left:right+1]

Step 6: Shrink the Window

Shrinking the window means removing a character from left in this context. So we will remove a character from chars_taken as well. And finally update the window bound left.

1
2
3


# Try to shrink window 
chars_taken[text[left]] -= 1
left += 1

Step 7: Return Smallest Substring

After the while loops ends, the best_text is guaranteed to be the smallest possible substring. But, the problem statement does not guaranteed that there will be a valid substring in text. So we need to check whether there is a valid substring or not before returning best_text.

Before the loops, best_text had a dummy string with length longer than the given string. If it is not updated inside the sliding window, that means best_text will still have that large dummy string. So we can check if length is greater than length of text and then return accordingly.

1
2
3
4
5


# if no string found
if len(best_text) <= len(text):
 return best_text
else:
 return ""

Every sliding window problem takes O(n) time itself, where n is the length of text. Inside the loop, we are making a look-up in a counters which is implemented by hashtables and incrementing and decrementing variables. The intersection of counters however will take O(S) time where S is the minimum number of elements in best_text(the answer). It can be told that S is very much negligible compared to n. So the time complexity of the algorithm will remain O(n) and space complexity is also O(n) in the worst case. The program is given below.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28


from collections import Counter

class Solution:
 def minWindow(self, text: str, characters: str) -> str:
 chars_given = Counter(characters) # count characters
 chars_taken = Counter() # count taken characters

 best_text = "-" * (len(text) + 1) # bigger than text

 left = 0 # Window left
 for right, char in enumerate(text):
 chars_taken[char] += 1 # Expand window

 # While window can be shrunk (operator & is intersection)
 while chars_taken & chars_given == chars_given:
 # Update best_text
 if right - left < len(best_text):
 best_text = text[left:right+1]

 # Try to shrink window 
 chars_taken[text[left]] -= 1
 left += 1

 # if no string found
 if len(best_text) <= len(text):
 return best_text
 else:
 return ""

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44


class Solution {
 unordered_map<char, int> charsGiven; // count given characters
 unordered_map<char, int> charsTaken; // count taken characters
public:
 string minWindow(string text, string characters) {

 for(char character: characters)
 charsGiven[character]++;

 string bestText(text.length() + 1, (char)0); // Initialize with invalid state

 int left = 0;
 for(int right = 0; right < text.length(); right++) {
 char character = text[right];
 charsTaken[character]++;
 // While window can be shrunk
 while(isValidWindow()) {
 // Update bestText
 if(right - left < bestText.length()) {
 bestText = text.substr(left, right-left+1);
 }

 // Try to shrink window
 charsTaken[text[left]]--;
 left++;
 }
 }
 // If no string found, bestText will remain invalid
 if(bestText.length() > text.length())
 return "";
 return bestText;
 }

private:
 bool isValidWindow() {
 for(auto charCountPair: charsGiven) {
 if(charsTaken.find(charCountPair.first) == charsTaken.end())
 return false;
 if(charsTaken[charCountPair.first] < charsGiven[charCountPair.first])
 return false;
 }
 return true;
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44


class Solution {
 Map<Character, Integer> charsGiven;
 Map<Character, Integer> charsTaken;

 public String minWindow(String text, String characters) {
 charsGiven = new HashMap<>(); // count given characters
 charsTaken = new HashMap<>(); // count taken characters

 for(char character : characters.toCharArray())
 charsGiven.put(character, charsGiven.getOrDefault(character, 0) + 1);

 String bestText = new String(new char[text.length()+1]); // Initialize with invalid state

 int left = 0;
 for(int right = 0; right < text.length(); right++) {
 char character = text.charAt(right);
 charsTaken.put(character, charsTaken.getOrDefault(character, 0) + 1);
 // While window can be shrunk
 while(isValidWindow()) {
 // Update bestText
 if(right - left < bestText.length())
 bestText = text.substring(left, right+1);

 // Try to shrink window
 charsTaken.put(text.charAt(left), charsTaken.getOrDefault(text.charAt(left), 0) - 1);
 left++;
 }
 }
 // If no string found, bestText will remain invalid
 if(bestText.length() > text.length())
 return "";
 return bestText;
 }

 private boolean isValidWindow() {
 for(char character: charsGiven.keySet()) {
 if(!charsTaken.containsKey(character))
 return false;
 if(charsTaken.get(character) < charsGiven.get(character))
 return false;
 }
 return true;
 }
}

Leetcode Problem - Monotonic Array

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/monotonic-array/
Difficulty: Medium
Category: Arrays

Step 1: Grasping the Problem

The median of a finite list of numbers is the “middle” number, when those numbers are listed in order from smallest to greatest.

If there is an odd number of numbers, the middle one is picked. For example, consider the list of numbers

[1, 3, 3, 6, 7, 8, 9]

This list contains seven numbers. The median is the fourth of them, which is 6.

If there is an even number of observations, then there is no single middle value; the median is then usually defined to be the mean of the two middle values.[1][2] For example, in the data set

[1, 2, 3, 4, 5, 6, 8, 9]

Step 2: Building data structures concept with heaps

A binary heap is an abstract binary tree with the following properties:

A child parent relation will always hold in the whole tree. Most of the time, the relation is lesser than (<) or greater than (>).
It must be a complete binary tree. A complete binary tree is a binary tree in which every level, except possibly the last, is completely filled, and all nodes are as far left as possible.

Below are two representations of a min-heap (a heap where the child-parent relationship is “<”) and a max-heap (a heap where the child-parent relationship is “>”).

1
2
3
4
5
6


 1 10
 / \ / \
 5 3 7 3
 / / \ / \
7 3 5 1 -10
 min-heap max-heap

The above two heaps can be represented in arrays like below:

1
2


min-heap: [1, 5, 3, 7]
max-heap: [10, 7, 3, 3, 5, 1, -10]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


import heapq

an_array = []

heapq.heappush(an_array, 5)
heapq.heappush(an_array, 3)
heapq.heappush(an_array, 6)
heapq.heappush(an_array, 1)
heapq.heappush(an_array, 9)

print(an_array)
# [1, 3, 6, 5, 9]

print(heapq.heappop(an_array))
# 1
print(heapq.heappop(an_array))
# 3

print(an_array)
# [5, 9, 6] # 5 is top of tree

Step 3: `addNum(int)` implementation

1
2
3
4


def __init__(self):
 # Initialization
 self.left = [] # A minheap top is left[0]
 self.right = [] # A maxheap top is -right[0]

Python does not have any implementation for max-heaps. But it can be achieved with a very simple trick. Just negate the values before pushing them to the heap!

We will push the new values to the left first, then pop that value back and push it to the right heap. This makes the left and right arrays always heapified.

1
2
3


def addNum(self, num: int) -> None:
 heapq.heappush(self.left, num)
 heapq.heappush(self.right, -heapq.heappop(self.left))

1
2
3



if len(self.left) < len(self.right):
 heapq.heappush(self.left, -heapq.heappop(self.right))

This completes the addNum(int) function implementation.

Step 4: `findMedian()` implementation

1
2
3
4


def findMedian(self) -> float:
 if len(self.left) > len(self.right):
 return self.left[0]
 return (self.left[0] - self.right[0]) * 0.5

The complexity of addNum(int) is O(logn) because we are inserting a value in a binary tree. The complexity of findMedian(self) is constant.

The final solution is given below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


import heapq

class MedianFinder:
 def __init__(self):
 # Initialization
 self.left = [] # A minheap top is left[0]
 self.right = [] # A maxheap top is -right[0]


 def addNum(self, num: int) -> None:
 heapq.heappush(self.left, num)
 heapq.heappush(self.right, -heapq.heappop(self.left))

 if len(self.left) < len(self.right):
 heapq.heappush(self.left, -heapq.heappop(self.right))

 def findMedian(self) -> float:
 if len(self.left) > len(self.right):
 return self.left[0]
 return (self.left[0] - self.right[0]) * 0.5

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31


class MedianFinder {
 priority_queue<int> left;
 priority_queue<int, vector<int>, greater<int> > right;
public:
 /** initialize your data structure here. */
 MedianFinder() { }

 void addNum(int num) {
 left.push(num);
 right.push(left.top());
 left.pop();

 if(left.size() < right.size()) {
 left.push(right.top());
 right.pop();
 }
 }

 double findMedian() {
 if(left.size() > right.size())
 return left.top();
 return (left.top() + right.top()) * 0.5;
 }
};

/**
 * Your MedianFinder object will be instantiated and called as such:
 * MedianFinder* obj = new MedianFinder();
 * obj->addNum(num);
 * double param_2 = obj->findMedian();
 */

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35


import java.util.*;
import java.lang.*;

class MedianFinder {
 /** initialize your data structure here. */
 PriorityQueue<Integer> left;
 PriorityQueue<Integer> right;
 public MedianFinder() {
 left = new PriorityQueue<Integer>(); // Java min-heap
 right = new PriorityQueue<Integer>( // Java max-heap
 (Integer val1, Integer val2) -> (- Integer.compare(val1, val2))
 );
 }

 public void addNum(int num) {
 left.add(num);
 right.add(left.remove());

 if(left.size() < right.size())
 left.add(right.remove());
 }

 public double findMedian() {
 if(left.size() > right.size())
 return left.peek();
 return (left.peek() + right.peek()) * 0.5d;
 }
}

/**
 * Your MedianFinder object will be instantiated and called as such:
 * MedianFinder obj = new MedianFinder();
 * obj.addNum(num);
 * double param_2 = obj.findMedian();
 */

Leetcode Problem - Parallel Courses

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/parallel-courses/
Difficulty: Hard
Category: Graphs

Step 1: Grasping the Problem

There are N courses with a prerequisites relationship among them. Our task is to determine the number of semester needed to complete all the courses. Here are the key points that will help us get started:

We cannot take a course whose prerequisites are not yet taken.
We cannot take a course whose prerequisites are even taken in this semester.
We have to start with the course that has no prerequisites.
If there is a cycle in the prerequisites relationship, there will be no solution.

As there are relationship among courses and we need to pick and complete each course at semesters, we know that it is a graph problem. We need to check if there is any cycle or not, if there isn’t, then we will need to get the course whose prerequisites are all complete. This is exactly what Topological Sorting does.

In computer science, a topological sort or topological ordering of a directed graph is a linear ordering of its vertices such that for every directed edge uv from vertex u to vertex v, u comes before v in the ordering.

So the answer of our problem is to make a topological ordering of the courses. And if we cannot make the ordering, then we will need to return -1.

Step 2: Build the Graph

To solve the problem with topological sorting, we need a graph first. In this graph, the nodes will be courses and the edges will be the relationships. The relationships are already given as edges relations. We will convert these into adjacency matrix representation.

So the graph will be a dictionary where, keys are courses and values are lists having courses that are dependent on the keys. We loop through relations and put them in the graph accordingly.

1
2
3
4
5
6


graph = {}
for relation in relations:
 # Make sure we don't get KeyError
 if relation[0] not in graph:
 graph[relation[0]] = []
 graph[relation[0]].append(relation[1])

Step 3: Topological Sorting Initialization

Topological Sorting requires us to track the in degrees of all nodes. For this, we will create an array named in_degree of size N+1. The courses in the problem is given in 1-indexed notation, for this, we have to create the extra unused index at 0.

1
2


in_degree = [0] * (N+1)
# Or: in_degree = [0 for _ in range(N+1)]

We can track the degrees while building the graph. Each time we find a relation, we increase the in degree of the descendant node.

1
2
3
4
5
6
7
8


# in_degree of N nodes, 1 indexed
in_degree = [0]*(N+1) # in_degree initialization
graph = {}
for relation in relations:
 if relation[0] not in graph:
 graph[relation[0]] = []
 graph[relation[0]].append(relation[1])
 in_degree[relation[1]] += 1 # update in_degree

Topological Sorting is very similar to BFS (Breadth First Search) or DFS (Depth First Search). Except that BFS/DFS starts from a source, topological sorting starts from all the nodes that has in_degree = 0. We will do a BFS. To do this, we will need a queue. For implementation, we will use pythons built-in collections.deque data structure. Start a loop through all the N courses, and put the courses that has in_degree = 0 into the queue.

1
2
3
4
5


# Collect all 0 in degree nodes
queue = deque()
for i in range(1, N+1):
 if in_degree[i] == 0:
 queue.append(i)

Step 4: Topological Sorting

Topological Sorting will keep running a loop until the queue is empty. At the beginning of the loop, we will keep track of how many courses we have in the queue. Because that will be the number of courses we can take in this semester (this_semester). this_semester is not used in anywhere, it is named so, for understanding purposes only.

At each semester, we will take each course from the queue, and decrease the in_degrees of all the courses that depend on current course. If in_degree for a course becomes 0, we push it into the queue to mark it available for the next semester. After doing this for all the queue elements, we will increament the semester by 1.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15


# Topological sort
semesters = 0
courses_taken = 0
while queue:
 # len(queue) courses in 1 semester
 for this_semester in range(len(queue)):
 course = queue.popleft()
 courses_taken += 1

 if course in graph:
 for neighbor in graph[course]:
 in_degree[neighbor] -= 1
 if in_degree[neighbor] == 0:
 queue.append(neighbor)
 semesters += 1

courses_taken another variable is used to count the number of courses popped from the queue. This will be used after the topological sorting to test if we could take all the courses or not. Here, courses_taken != N will be True when there is a cycle detected in the graph and there is no topological sorting possible.

Below is a demonstration of the topological sorting process.

If courses_taken == N, we know that we have taken all the courses, so we return the number of semesters.

1
2
3
4
5


# Make sure all courses are taken
if courses_taken != N:
 return -1

return semesters

This is a pure algorithm of topological sorting with no overhead. So the time and space complexity for this problem is that of topological sorting, O(N+|relations|) and O(N). The complete solution is given below.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42


from collections import deque
from typing import List

class Solution:
 def minimumSemesters(self, N: int, relations: List[List[int]]) -> int:
 # in_degree of N nodes, 1 indexed
 in_degree = [0]*(N+1)
 graph = {}
 for relation in relations:
 # Make sure we don't get KeyError
 if relation[0] not in graph:
 graph[relation[0]] = []
 graph[relation[0]].append(relation[1])
 in_degree[relation[1]] += 1

 # Collect all 0 in degree nodes
 queue = deque()
 for i in range(1, N+1):
 if in_degree[i] == 0:
 queue.append(i)

 # Topological sort
 semesters = 0
 courses_taken = 0
 while queue:
 # len(queue) courses in 1 semester
 for this_semester in range(len(queue)):
 course = queue.popleft()
 courses_taken += 1

 if course in graph:
 for neighbor in graph[course]:
 in_degree[neighbor] -= 1
 if in_degree[neighbor] == 0:
 queue.append(neighbor)
 semesters += 1

 # Make sure all courses are taken
 if courses_taken != N:
 return -1

 return semesters

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46


class Solution {
public:
 int minimumSemesters(int N, vector<vector<int>> &relations) {
 // in_degree of N nodes, 1 indexed
 vector<int> inDegree(N+1);
 map<int, vector<int>> graph;
 for(auto relation: relations) {
 graph[relation[0]].push_back(relation[1]);
 inDegree[relation[1]]++;
 }

 // Collect all 0 in degree nodes
 queue<int> q;
 for(int i = 1; i < N+1; i++)
 if(inDegree[i] == 0)
 q.push(i);

 // Topological sort
 int semesters = 0;
 int coursesTaken = 0;
 while(!q.empty()) {
 // q.size() courses in 1 semester
 int thisSemesterCourses = q.size();
 for(int thisSemester = 0; thisSemester < thisSemesterCourses; thisSemester++) {
 int course = q.front();
 q.pop();
 coursesTaken++;

 if(graph.find(course) != graph.end()) {
 for(auto neighbor: graph[course]) {
 inDegree[neighbor]--;
 if(inDegree[neighbor] == 0)
 q.push(neighbor);
 }
 }
 }
 semesters++;
 }

 // Make sure all courses are taken
 if(coursesTaken != N)
 return -1;

 return semesters;
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46


class Solution {
 public int minimumSemesters(int N, List<List<Integer>> relations) {
 // in_degree of N nodes, 1 indexed
 int[] inDegree = new int[N+1];
 Map<Integer, List<Integer>> graph = new HashMap<>();
 for(List<Integer> relation: relations) {
 List<Integer> descendants = graph.getOrDefault(relation.get(0), new ArrayList<>());
 descendants.add(relation.get(1));
 graph.put(relation.get(0), descendants);
 inDegree[relation.get(1)]++;
 }

 // Collect all 0 in degree nodes
 Queue<Integer> queue = new LinkedList<>();
 for(int i = 1; i < N+1; i++)
 if(inDegree[i] == 0)
 queue.add(i);

 // Topological sort
 int semesters = 0;
 int coursesTaken = 0;
 while(!queue.isEmpty()) {
 // queue.size() courses in 1 semester
 int thisSemesterCourses = queue.size();
 for(int thisSemester = 0; thisSemester < thisSemesterCourses; thisSemester++) {
 int course = queue.remove();
 coursesTaken++;

 if(graph.containsKey(course)) {
 for(int neighbor: graph.get(course)) {
 inDegree[neighbor]--;
 if(inDegree[neighbor] == 0)
 queue.add(neighbor);
 }
 }
 }
 semesters++;
 }

 // Make sure all courses are taken
 if(coursesTaken != N)
 return -1;

 return semesters;
 }
}

Leetcode Problem - Sudoku Solver

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/sudoku-solver/
Difficulty: Hard
Category: Backtracking

Step 1: Grasping the Problem

We have to design a [Sudoku] solver. The problem statement defined what is Sudoku and how it is played. The given board is always a fixed size of 9x9. So, the input is always a fixed board. In that case, whatever we do with the solution (unless we are doing terribly complicated things), it will be a O(1) solution.

In spite of the time complexity is O(1), we need to think of the total time the solution might take. For example, if you do a blind brute force, the total search space for the 9x9 board is going to be (9x9)^9 = 150094635296999121 ~ 1.5x10^17. So a naive brute force solution will not be feasible.

There are some restrictions on putting a value in an index in the board. So, we do not need to put all values from 0 to 9 to check if we can solve. This is a perfect example for Backtracking algorithms where you have a number of of constraints to check and if following the constraints for a particular set of values end-up in a dead end, then go back and change some of the values to try the next space.

Wikipedia has an amazing animation to demonstrate how backtracking works with sudoku.

Step 2: Implement Board Validation Function

Before implementation of the solver itself, first we need to create a function that can be used to see if we can put a value on an index in the board. The function will take the board, a row, a col and finally a val to put into that index of the board.

1

def validate_board(self, board, row, col, val):

The function will first check if there is same value as val in the same row or column in the board. We can check this easily by looping through that row and column once at a time.

1
2
3
4
5
6
7
8
9


# Check if duplicate in row
for row_it in range(9):
 if board[row_it][col] == val:
 return False

# Check if duplicate in col
for col_it in range(9):
 if board[row][col_it] == val:
 return False

Then we will check if the same value as val also appears in the same 3x3 block. To do this, we will take the current block where row and col resides. Then we will iterate through the block and check accordingly.

1
2
3
4
5
6
7


# Check if block has duplicate
row_block = row - row%3
col_block = col - col%3
for row_it in range(3):
 for col_it in range(3):
 if board[row_block+row_it][col_block+col_it] == val:
 return False

Finally if none of these return False, we are sure that val at row, col position is a valid move. So we will return true.

1
2


# None of them return false. So satisfies
return True

Step 3: Base Conditions of Recursing Backtracking

We will start from the first element in the board. We will putting valid values in places and move forward. To do this, we will continuously put values, then increment col.

While incrementing col, we will reach at a point when col will be greater than 8 (out of the board). At that time, we reset col to zero and increment row. Finally row will go out of the board, so at that time, we will be sure that the board is solved.

1
2
3
4
5
6


if row == 9:
 # If all rows are complete, we have solved it!
 return True
if col == 9:
 # This column is complete, go to next row's first col
 return self.recursiveSolve(board, row+1, 0)

Also, if an index of the board is already filled by the problem, then we do not need to do anything at that index. So we will move forward.

1
2
3


if board[row][col] != '.':
 # This index is already filled
 return self.recursiveSolve(board, row, col+1)

Step 4: Implement Recursive Backtracking

For the backtracking, we will take numbers from 1 to 9 and fill each index of the board one by one. The problem gives us the board with characters in the indices. So we also need to put characters (or strings for python). For this, we will loop over the characters from ‘1’ to ‘9’.

1

for val in ['1', '2', '3', '4', '5', '6', '7', '8', '9']:

A more pythonic way of doing this is following.

1

for val in '123456789':

Then we will see if we can put val in the index with the help of our previously implemented function validate_board.

1

if self.validate_board(board, row, col, val):

If we can, we will put the value in the board and recursively go to the next index. And after continuously doing this, finally we will get an answer if puting val in this index results in a feasible board solution. If it is feasible, then we are done. If not, then we will have to backtrack. For backtracking we will again put a '.' in the index and continue with the next val.

1
2
3
4
5
6
7


# Put a correct value
board[row][col] = val
# Check if we can proceed to finish
if self.recursiveSolve(board, row, col+1):
 return True
# Couldn't finish, so BackTrack!
board[row][col] = '.'

Finally there will be at least one state for which, recursiveSolve will return True. For that case, we are returning True. And for the rest of the case, we will return False.

1
2


# Dead end in this path. So return false
return False

Step 5: Start the Backtracking

We will start the backtracking process from row=0 and col=0. Even if there may be already a value in that index, our backtracking function will take care of it.

1
2
3
4
5
6


def solveSudoku(self, board: [[str]]) -> None:
 """
 Modify board in-place
 """
 # Start solving from first index
 self.recursiveSolve(board, 0, 0)

This is the most known algorithm for solving Sudoku. The time complexity as already mentioned is O(1). The space is only the board size. The problem expects you to change the board in place so this solution is perfect. If you could not change the board and return a new one, you can do a slight modification to this program like copying the board before the backtracking and returning the board after the process.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


def solveSudoku(self, board: [[str]]) -> [[str]]:
 """
 Modify board in-place
 """
 # Copy board
 board_copy = board.copy()
 # Start solving from first index
 self.recursiveSolve(board_copy, 0, 0)
 # Return the copy
 return board_copy

The complete solution of the problem is given by the following.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54


class Solution:
 def solveSudoku(self, board: [[str]]) -> None:
 """
 Modify board in-place
 """
 # Start solving from first index
 self.recursiveSolve(board, 0, 0)


 def validate_board(self, board, row, col, val):
 # Check if duplicate in row
 for row_it in range(9):
 if board[row_it][col] == val:
 return False

 # Check if duplicate in col
 for col_it in range(9):
 if board[row][col_it] == val:
 return False

 # Check if block has duplicate
 row_block = row - row%3
 col_block = col - col%3
 for row_it in range(3):
 for col_it in range(3):
 if board[row_block+row_it][col_block+col_it] == val:
 return False
 # None of them return false. So satisfies
 return True

 def recursiveSolve(self, board, row, col):
 if row == 9:
 # If all rows are complete, we have solved it!
 return True
 if col == 9:
 # This column is complete, go to next row's first col
 return self.recursiveSolve(board, row+1, 0)
 if board[row][col] != '.':
 # This index is already filled
 return self.recursiveSolve(board, row, col+1)

 # Try putting all values in 1-9
 for val in '123456789':
 if self.validate_board(board, row, col, val):
 # Put a correct value
 board[row][col] = val
 # Check if we can proceed to finish
 if self.recursiveSolve(board, row, col+1):
 return True
 # Couldn't finish, so BackTrack!
 board[row][col] = '.'

 # Dead end in this path. So return false
 return False

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60


class Solution {
public:
 void solveSudoku(vector<vector<char>> &board) {
 // Start solving from first index
 recursiveSolve(board, 0, 0);
 }

private:
 bool validateBoard(vector<vector<char>> &board, int row, int col, int val) {
 // Check if duplicate in row
 for(int rowI = 0; rowI < board.size(); rowI++)
 if(board[rowI][col] == val)
 return false;

 // Check if duplicate in col
 for(int colI = 0; colI < board.size(); colI++)
 if(board[row][colI] == val)
 return false;

 // Check if block has duplicate
 int rowBlock = row - row%3;
 int colBlock = col - col%3;
 for(int rowI = 0; rowI < 3; rowI++)
 for(int colI = 0; colI < 3; colI++)
 if(board[rowBlock+rowI][colBlock+colI] == val)
 return false;

 // None of them return false. So satisfies
 return true;
 }

 bool recursiveSolve(vector<vector<char>> &board, int row, int col) {
 // If all rows are complete, we have solve it!
 if(row == 9)
 return true;

 // This column is done, go to next row's first col
 if(col == 9)
 return recursiveSolve(board, row+1, 0);

 // This index is already filled
 if(board[row][col] != '.')
 return recursiveSolve(board, row, col+1);

 // Try putting all values in 1-9
 for(char val = '1'; val <= '9'; val++) {
 if(validateBoard(board, row, col, val)) {
 // Put a correct value
 board[row][col] = val;
 // Check if we can proceed to finish
 if(recursiveSolve(board, row, col+1))
 return true;
 // Couldn't finish, so BackTrack!
 board[row][col] = '.';
 }
 }
 // Dead end in this path, So return false
 return false;
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58


class Solution {
 public void solveSudoku(char[][] board) {
 // Start solving from first index
 recursiveSolve(board, 0, 0);
 }

 private boolean validateBoard(char[][] board, int row, int col, int val) {
 // Check if duplicate in row
 for(int rowI = 0; rowI < board.length; rowI++)
 if(board[rowI][col] == val)
 return false;

 // Check if duplicate in col
 for(int colI = 0; colI < board.length; colI++)
 if(board[row][colI] == val)
 return false;

 // Check if block has duplicate
 int rowBlock = row - row%3;
 int colBlock = col - col%3;
 for(int rowI = 0; rowI < 3; rowI++)
 for(int colI = 0; colI < 3; colI++)
 if(board[rowBlock+rowI][colBlock+colI] == val)
 return false;

 // None of them return false. So satisfies
 return true;
 }

 private boolean recursiveSolve(char[][] board, int row, int col) {
 // If all rows are complete, we have solve it!
 if(row == 9)
 return true;

 // This column is done, go to next row's first col
 if(col == 9)
 return recursiveSolve(board, row+1, 0);

 // This index is already filled
 if(board[row][col] != '.')
 return recursiveSolve(board, row, col+1);

 // Try putting all values in 1-9
 for(char val = '1'; val <= '9'; val++) {
 if(validateBoard(board, row, col, val)) {
 // Put a correct value
 board[row][col] = val;
 // Check if we can proceed to finish
 if(recursiveSolve(board, row, col+1))
 return true;
 // Couldn't finish, so BackTrack!
 board[row][col] = '.';
 }
 }
 // Dead end in this path, So return false
 return false;
 }
}

Leetcode Problem - Tree Right Side

rahatzamancse@gmail.com (Rahat Zaman) — Mon, 25 May 2020 00:00:00 +0000

Problem link: https://leetcode.com/problems/binary-tree-right-side-view/
Difficulty: Medium
Category: Trees

Step 1: Grasping the Problem

This problem is a pure binary tree problem. You are given a binary tree with integer value in each node. Suppose that you are on the right side of the tree, so you need to print the values of the nodes from top to bottom which you will see.

1
2
3
4
5
6
7


 5 <------- 👁
 / \
 3 7 <----- 👁
 / \ / \
4 1 1 12 <-- 👁

result = [5, 7, 12]

A very misleading approach to solve this problem is thinking of always going to the right child and print them out. This is completely wrong and the problem requires you to visit all nodes to solve. See some examples below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25



 Tree 1

 5 <------- 👁
 / \
 3 7 <----- 👁
 /
 4 <-------------- 👁
 /
 9 <---------------- 👁

result = [5, 7, 4, 9]


 Tree 2

 5 <------- 👁
 / \
 3 7 <----- 👁
 / \
 4 6 <--------- 👁
 / /
 9 2 <----------- 👁

result = [5, 7, 6, 2]

For visiting all the nodes in the tree, we need a tree traversal algorithm. We are at the right side of the tree and we need to print from top to bottom. That means we must do two things while traversing the tree.

We must visit the parent before visiting it’s children. Because we need to print from top to bottom. This gives us an intuition of using a pre-order traversal.
We must visit the right child of two siblings at the same level. Because we need to see the rightmost node first. This gives us an intuition of traversing the right child first in the pre-order traversal.

So we have came up with a pre-order traversal visiting the right child first. But how are we going to make sure that the node is shadowed by another node at the name level (and at it’s right side)? To do this, we can use a level variable and check if any node has already been visited at that height. If visited, we will continue traversing the nodes children without using itself for the answer. But a more elegant solution is taking parts of the left that is after our stored node. It will be cleared more in the implementation step.

Step 2: Recursing pre-order traversal

A recursive pre-order traversal can be thought dividing into subproblems and solving them recursively. First we see if the current node is None of not. If current node is None, we will return an empty result List.

1
2


if not root:
 return []

This also gives up guarantee for accessing the nodes children. Now that we can are sure that current node is not None, we go to our right child (Remember, we must visit the right child first) and do this same thing. “Doing the same thing for another node” is similar to calling the same function (recursively) with different parameters.

1

right = self.rightSideView(root.right)

Finally we do the same thing for the left child.

1

left = self.rightSideView(root.left)

Step 3: Creating Visited Nodes List

At this point, the recursive calls have finished and we will have the final result from both the right and left children. We just need to aggregate both results into one final list.

Let us see the example again in the below tree.

 5(1)
/ \
3(3) 7(2)
/ \
(6)4 6(4)
/ /
(7)9 2(5)

This is the breakdown of each recursive call and the list we want to return from those calls.

Calls that needs to return	Final Return	Root	Right	Left
call 1	[5, 7, 6, 2]	[5]	[7 ]	[6, 2]
call 2	[7 ]	[7]	[ ]	[ ]
call 3	[6, 2 ]	[ ]	[6, 2]	[ ]
call 4	[6, 2 ]	[6]	[ ]	[ ]
call 5	[2 ]	[2]	[ ]	[ ]
call 6	[ ]	[ ]	[ ]	[ ]
call 7	[ ]	[ ]	[ ]	[ ]

It is clear the every time we are returning the root itself first. And after that, we are also returning the result of right calls. So clearly for these two, the code will be something like this:

1

return [root.val] + right + # and the rest

We are wrapping the root.val so that python can concatenate the list for us. You could also do [root.val, *right] if you want, the result is the same.

Finally, we need to add the rest nodes (coming from the left subtrees). We can clearly tell that it will be a suffix of the left child’s recursive call’s return list. We know that if we have found n children from the right recursive call, we do not need the first n children from the left call (because they are shadowed by the right nodes). So we can remove first len(right) nodes from the left list and concatenate that with our result.

1

return [root.val] + right + left[len(right):]

This is it. The complete program just loops through the nodes recursively and at the end concatenates creating the list. The concatenation of python lists takes amortized O(1) time. So the complete time complexity is O(n) where there are n nodes in the tree. The space complexity is also O(n) in both average and worst case. The program is very small in python and given below.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14


# Definition for a binary tree node.
class TreeNode:
 def __init__(self, val=0, left=None, right=None):
 self.val = val
 self.left = left
 self.right = right

class Solution:
 def rightSideView(self, root: TreeNode) -> [int]:
 if not root:
 return []
 right = self.rightSideView(root.right)
 left = self.rightSideView(root.left)
 return [root.val] + right + left[len(right):]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26


/**
 * Definition for a binary tree node.
 * struct TreeNode {
 * int val;
 * TreeNode *left;
 * TreeNode *right;
 * TreeNode() : val(0), left(nullptr), right(nullptr) {}
 * TreeNode(int x) : val(x), left(nullptr), right(nullptr) {}
 * TreeNode(int x, TreeNode *left, TreeNode *right) : val(x), left(left), right(right) {}
 * };
 */
class Solution {
public:
 vector<int> rightSideView(TreeNode* root) {
 if(root == nullptr)
 return {};
 vector<int> right = rightSideView(root->right);
 vector<int> left = rightSideView(root->left);
 vector<int> result;
 result.push_back(root->val);
 result.insert(result.end(), right.begin(), right.end());
 for(int level = right.size(); level < left.size(); level++)
 result.push_back(left[level]);
 return result;
 }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29


/**
 * Definition for a binary tree node.
 * public class TreeNode {
 * int val;
 * TreeNode left;
 * TreeNode right;
 * TreeNode() {}
 * TreeNode(int val) { this.val = val; }
 * TreeNode(int val, TreeNode left, TreeNode right) {
 * this.val = val;
 * this.left = left;
 * this.right = right;
 * }
 * }
 */
class Solution {
 public List<Integer> rightSideView(TreeNode root) {
 if(root == null)
 return new ArrayList<Integer>();
 List<Integer> right = rightSideView(root.right);
 List<Integer> left = rightSideView(root.left);
 List<Integer> result = new ArrayList<Integer>();
 result.add(root.val);
 result.addAll(right);
 for(int level = right.size(); level < left.size(); level++)
 result.add(left.get(level));
 return result;
 }
}

Machine Learning Tutorial - Lesson 01

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Introduction to Machine Learning

Introduction

In this series, I am presenting to you, the hottest topic of this era, Machine Learning. Throughout this series, you will be starting from scratch, and slowly learn from basic to advanced algorithms of Machine Learning. We will walk you through a lot of popular Machine Learning frameworks like NumPy, Scikit-learn, TensorFlow, Keras, etc. So get ready to start your awesome ride to the world where Computers (Machines) are treated as children and we teach them many things with real-world data.

We will also learn how to understand and manipulate data. I will put effort on creating complete pipeline for your data from preprocessing to model evaluation. After this series, you can do extensive exploratory data analysis to understand and present different summary and visualization of data like a chart given below:

What is Machine Learning?

Whenever people think of Machine Learning, they imagine a robot who looks very similar to a human and act like a human. It has its own heart and mind, can take decisions, and be intelligent. But actually, machine learning is not necessarily a robot/agent. It can be any device or even only software with no hardware tools (we will mostly build a machine learning algorithm/software in this series). The special characteristic of Machine Learning-based software is that it can react to new kinds of inputs and give output based on that new inputs or instructions.

In a typical programming world, we create an application with a programming language, using different kinds of logic like conditionals, loops, etc. In traditional programming, we give the program a set of formulas or rules, so it can take a limited set of inputs and give a set of fixed outputs. Programming a machine learning algorithm is slightly different from traditional programming. In Machine Learning, we provide a program a set of inputs (and probably their outputs) and expect that it will work on a new set of inputs (that is never seen or expected by the program). When a program can look at data, and then become able to analyze a new set of data, it is Machine Learning.

According to Wikipedia:

Machine learning (ML) is the study of computer algorithms that improve automatically through experience and by the use of data.

So like the above image, you do not need to code the rules of adding two numbers. You can only provide a bunch of examples for addition (e.g. 1+2=3, 4+3=7 etc.), and the program will learn how to do addition by itself!

Prerequisites

What do you need before getting started with this Machine Learning series? You can do ML with any programming language. Most of the programming languages have frameworks/libraries for machine learning. But the most popular ones right now are Python and R programming languages. For this series, we will go with Python because it is used more than R.

Moreover, you will need to know Math. Specifically, to understand the core concepts of Machine Learning like gradient descent algorithm, forward feeding, and backpropagation algorithm, you will need to know Calculus. But for the sake of simplicity, generality, and most importantly fun, we will try to remove most of the underlying concepts of ML, and focus on the application and usage of pre-built ML algorithms. So do not worry about math too much if you don’t want it right now.

Applications of Machine Learning

1. Product Recommendations: Currently the most widely used application of ML is recommendation systems. Almost every big tech company uses this for their product advertisements.

2.Image Recognition: Did you ever think that computers will be able to understand scenery? Now it can identify each object of a scene and make decisions just like humans.

3.Speech Recognition: Ever wondered how your smartphone assistant (Siri, Google Assistant, Alexa) can understand your speaking and talk to you? It is also an application of ML. More specifically, machine learning algorithms with sequential data handling capabilities can work for speech recognition and synthesis.

4. Self Driving Car: Self Driving Car will be the future. To make it happen, a collaboration from Image Recognition to Pathfinding and decisions making. All of these are possible with the help of ML.

5. Fraud Detection: ML can easily detect outliers from your data. Using online fraud detection, many bank transaction systems can be kept safe. We will learn how to detect outliers in your data and preprocess it so your data becomes clean.

6. Image synthesis: ML can be used to generate new images from existing images. With the help of Generative Adversarial Networks, all of these cartoon charters are generated.

7. Story Generation: Using advanced Natural Language Processing (NLP) techniques, now computers can create beautiful new stories that are very pleasing to hear.

8. Content writing assistant: Have you ever heard of the online tool Grammarly? It is a wonderful tool that helps you correct your spelling and grammar mistakes. It is solely based on NLP and Machine Learning.

9. Translation: People now can understand any language with the help of their smartphones. This is also possible because of text translation with Machine Learning. Now you can just hold your camera to a foreign-language sign, and it will capture the sign, convert it into text and then translate it for you. We will have a look at OCR in one of our upcoming lessons.

10. Streaming Video Compression: Last but not least, with machine learning high-quality video conferencing is possible. An HD web camera can produce up to 1 Gigabyte of data within seconds. Imagine how much data is needed to be passed through the internet when you are sitting in an online class with 30 more students. But you are getting high-quality video because, with ML, the video is being compressed to kilobytes and then sent through the internet. Image, Audio, and Video compression are also done with ML nowadays.

Brief overview of Python

If you think that you became rusty in python, or don’t have the confidence to go deep into ML without sharpening your python skills, here I will cover the fundamentals of python. You can skip this section if you are confident enough. Below are the top things of python we will need throughout the series:

Importing

As said previously, we will be using a lot of libraries for ML. So we start with importing all those libraries. In python, it is always good practice to import all the required libraries at the top of the .py file.

1
2


import cv2 # Import the popular computer vision library OpenCV
import numpy as np # Importing the library named Numeric Python (NumPy) as np

You can install new libraries through a package manager like pip or anaconda. For example, to install numpy, you simply type in the console:

1

pip install numpy

Stating Variables

Variables are placeholders for different kinds of values. Python is a dynamically typed language, which means one variable can be used to hold different types of values. Python has the following four primitive types for variables:

Integer: An integer can hold a whole number. Python supports big integers by default, so you can keep as big an integer as you want in a variable.
Float: A float is much like an integer, but holds a rational value.
Bool: A Boolean value is a two-state value. True or False.
Str: A String type holds characters or Unicode values.

Below is an example to use all the built-in types of variables:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18


an_integer = 5
a_float = 5.5
a_bool = True
an_str = 'ML'
another_str = "ML"
yet_another_multiline_str = '''This
Multiline
string'''

# This is a comment
# You can initialize multiple variables in one line with pattern matching
a, b = 5, 3.6

# Assignment statement returns the same value back. So we can initialize multiple variables like this
a = b = 5

# To know a type of a variable, use the type() function
print(type(a)) # int

Operations

Below is a snippet that shows all common operations that can be done in python:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33


a = 5
b = 3

# Arithmetic
print(a + b) # 8
print(a - b) # 2
print(a * b) # 15
print(a / b) # 1.6666666666666667 (floating point division)
print(a // b) # 1 (integer division)
print(a % b) # 2 (modulus)
print(a ** b) # 125 (power)

# Bitwise
print(~a) # -6 (bitwise inversion)
print(a & b) # 1 (bitwise and)
print(a | b) # 7 (bitwise or)
print(a ^ b) # 6 (bitwise xor)
print(a << b) # 40 (bitwise left shift)
print(a >> b) # 0 (bitwise right shift)
## all operators appended with '=' makes it assignment operator

# Comparison operator
print(a == 5 and b == 5) # False (logical and)
print(a == 5 or b == 5) # True (logical or)
print(not (a == 5 or b == 5)) # False (logical not)

# Identity
print(a is b) # False (a and b are the same object)
print(a is not b) # True (a and b are different object)

# Membership operators
print(a in [1, 2, 3, 4, 5]) # True
print(a not in [1, 2, 3, 4, 5]) # False

Complex types

Besides the primitive types, python also has many classes predefined. Two of the most important classes among them are list and tuple. The type str discussed previously is also a subtype of the class list. The key difference among lists and tuples is that lists are mutable, and tuples are immutable. Here is a demonstration of both of them:

1
2
3
4
5
6
7


a_list = [1, 2, 3]
a_tuple = (1, 2, 3)

a_list[1] = 5
print(a_list) # [1, 5, 3]

a_tuple[1] = 5 # TypeError: 'tuple' object does not support item assignment

Conditionals

Like all other programming languages, python has conditionals like the following:

1
2
3
4


if 5 == 5:
 print('this will be always true')
else:
 print('We will never get here')

But unlike most languages, you can use short conditionals to get a value in one like

1

a = 5 if 15 % 3 == 0 else 10 # a = 5, because 15 is divisible by 3

Functions

Functions in python can receive several parameters and can return any value. They can also use variables declared in the global scope.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13


# Global scope variable
is_divisible = False

# declaring function
def divide(val, div):
 global is_divisible # Telling that we are using a global variable
 if val % div == 0:
 is_divisible = True
 # Returning an int
 return val // div
 else:
 # Returning a tuple (int, int)
 return val // div, val % div

Loops

The while loops in python are similar to all other programming languages:

1
2
3
4
5
6


i = 0
while i <= 10:
 # printing the value i. end parameter is printed after printing the value, whose default is '\n'
 print(i, end=' ')
 i += 1
# 0 1 2 3 4 5 6 7 8 9 10

Python does not have a for loop with “initialization”, “condition” and “statement” like C, C++, Java, JS, and many other languages. The only way for loops in python to work is through a list or generator. Let us look at a list first.

1
2
3
4


a = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
for i in a:
 print(i, end=' ')
# 0 1 2 3 4 5 6 7 8 9 10

Generators

Python has a special kind of data type named generator. A generator is like a function but keeps its state after returning something to resume execution for consecutive calls. Let us create a generator that yields ints from a starting value to an ending value.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


def range_generator(start, end):
 i = start
 while i != end:
 yield i
 i += 1

for i in range_generator(0, 5):
 print(i, end=' ')

# 0 1 2 3 4

Fortunately, Python provides a very helpful built-in generator function named range. It can be used to create many types of lists. Let us see some example:

1
2
3
4
5
6
7
8
9


# create a list of even numbers from 0 to 10
for i in range(0, 11, 2):
 print(i)
# 0, 2, 4, 6, 8, 10

# create a list of even numbers from 0 to 10 but reversed
for i in range(10, -1, -2):
 print(i)
# 10, 8, 6, 4, 2, 0

List comprehensions

Python has a list/tuple creator syntax, that can be used to create many complex types of lists with only one line. For this feature, python is known as ** one-liner programming language**.

1
2
3


list_of_evens = [i for i in range(0, 11, 2)]
list_of_another_evens = [i*2 for i in range(6)]
# Both are: 0, 2, 4, 6, 8, 10

Pattern Matching feature

Python has an awesome pattern matching feature that can be used to save a lot of time and effort while creating efficient understandable code. Let us see a demonstration of pythons pattern matching:

1
2
3
4
5
6


three_numbers = [1, 2, 3]

def add_three_numbers(a, b, c):
 return a + b + c

print(add_three_numbers(*three_numbers)) # 6

Here, the list three_numbers is expanded with the * operator, and then matched sequentially with a, b, and c. In future lessons, you will see a lot of functions like below:

1
2
3
4


def a_functions(param1, param2, *args, **kwargs):
 an_option_without_name = args[0]
 an_option_with_name = kwargs['an_option']
 # Rest of the body

Here, args and kwargs are not any specific keywords. You can use any other name if you want. It is just used by convention. If a function can evolve over time (with different release versions of the library), it is a good idea to keep the args and kwargs so later we can get any parameters in the newer version without breaking backward compatibility.

Also, another example of using args is shown with the summing function.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


def sum(*args):
 if not args:
 return 0
 total = args[0]
 # We will understand list slicing within a minute. The below syntax means that we are skipping the first index
 for i in args[1:]:
 total += i
 return total

print(sum(1, 2, 3, 4)) # 10

List slicing

We can slice a list/str pretty easily with the list slicing syntax. The syntax is like list[inclusive_start:exclusive_end:jump]. You will get the gist with the example below:

1
2
3
4
5
6
7
8


# Create a generator to get from 0 to 9.
# Then convert it into list
a = list(range(10)) # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

print(a[5:8]) # [5, 6, 7]
print(a[5:]) # [5, 6, 7, 8, 9]
print(a[:8]) # [0, 1, 2, 3, 4, 5, 6, 7]
print(a[3:8:2]) # [3, 5, 7], get every second element starting from index 3 till index 8

Remember, these are the bare-bone basics of the wonderful python language. This will give you a quick start with python if you have programming knowledge with any other programming language or you think you need to sharpen the basics of python before this series. There are a lot more than just these with python. But with this given knowledge, a simple google search will help you understand easily if you encounter a new syntax.

Setting up the Machine Learning environment

Now that you have become more confident to work with python, let us set up the environment for all the upcoming lessons. The best practice to do data science in python is to use a version manager. You can use anything like Anaconda, Virtualenv, or even the built-in pip if you are comfortable with it. A version manager is necessary because you may need a specific version of python to do a specific experiment with a package. If you directly install python to your machine, then you will not be able to use separate versions for your different experiments.

Go ahead, download, and install Anaconda. If you are low on resources, you can try the miniconda. After installing, add the bin folder from the installed directory into your PATH variable. You can follow the official guide from Microsoft to do so. After adding to PATH, verify that you can use it in the CMD. Open a CMD and type:

1

conda -V

If the command works, you have successfully installed conda. Anaconda is a python environment manager. It can manage different python versions and their libraries separately in different environments. Let us now create a python environment using anaconda with python version 3.8.

1

conda create -n newenv python=3.8

After that, you can activate the environment, so whenever you run a python file, it will be interpreted with the python interpreter from the newenv.

1

conda activate newenv

Now write a python file with print('hello world') and save it as main.py. Then run it with python main.py (remember to activate the newenv environment). You can see that it works and the python version used will be the python in your environment. Finally, let us install some useful packages that we will use throughout all the upcoming lessons. Type the following command in CMD.

1

conda install -n newenv numpy pandas jupyterlab

Let us dissect the command part by part. The first word conda is the executable name. Then we specify a command install indicating conda that we want to install something. Then we provide an argument -n with the value newenv. -n newenv indicates that we want to install something into the environment newenv. After that, we provide three packages numpy, pandas, and jupyterlab. Press Y where necessary and you have installed 3 new packages into your environment named newenv. Let’s test those packages now.

Jupyterlab (or jupyter notebook) is an awesome tool for data science and machine learning enthusiasts. It gives you so much flexibility that you will not want to leave that soon. Activate the environment and open a jupyterlab instance from CMD.

1
2


conda activate newenv
jupyter-lab

Your default browser will open with a localhost page like below:

Press the “Python 3” button under the Notebook section and you will create a new notebook file in your current directory.

Now in the first cell, type the following python code:

1
2
3
4


import numpy as np
arr = [1, 2, 3, 4]

np.sum(arr)

Then press Shift+Enter to run the cell. You will get the sum of the list, 10.

Now let us explain what you just did. Jupyterlab or Jupyter notebook creates python kernels that run for each notebook. Notebooks are file formats (written in JSON) that use python kernels to run. When you run a notebook cell in jupyter notebook, the state of the cell is saved in the python kernel. That means all the variables and functions declared there will be available throughout the notebook.

For the rest of the series, it is up to you to decide which method to use for your data science career. We will only discuss the python codes you need. You can run that code in a notebook or as a file.

Conclusion

In this lesson, we warmed up ourselves with python and set up our environment for Machine Learning. From the next lesson, we will step into the field of ML. Until then, I will recommend you to play around with python, anaconda, and jupyter notebook.

Machine Learning Tutorial - Lesson 02

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

All about Numpy and Pandas

Introduction

NumPy and Pandas are the most popular libraries for numeric computation in python. NumPy is so popular that there are debates that the library should be built into python itself and not a separate module. In this lesson, we will walk through many useful features that these two giant libraries offer. After this lesson, you will be very comfortable with both of these libraries.

NumPy’s calculative protocols are so popular that, popular Machine Learning frameworks like TensorFlow and PyTorch follow this protocol into their own tensor computation. They also provide methods to retrieve tensors as NumPy arrays. So learning this will help you to use almost all other advanced modules we will introduce in future lessons. Finally, in your upcoming Machine Learning career, you will never have trouble using these two libraries.

Basics of Matrix

Before manipulating data in NumPy, you need some fundamental knowledge of matrix. A matrix is a collection of numbers arranged into a fixed number of rows and columns. It can also be thought of as a grid of numbers. Below is an example of a matrix:

In python, you can define a matrix using a two-dimensional list. We will go through dimensionality later in this lesson, but for now, think of a two-dimensional list as a list within a list. See the above matrix implemented in python below:

1
2
3
4
5
6
7
8
9


a_matrix = [
 [1, 2, 3, 4],
 [8, 7, 6, 5],
 [9, 10, 11, 12]
]

# Accessing elements

print(a_matrix[1][2]) # 6

Remember, a matrix should always have equal rows in all columns, and equal columns in all rows. b_matrix in code below is not a matrix:

1
2
3
4
5
6


# not a matrix
b_matrix = [
 [1, 2, 3, 4],
 [8, 6, 5], # This row has 3 columns and the rest has 4
 [9, 10, 11, 12]
]

Operations in Matrix

Addition and subtraction

Addition and subtraction in matrices are always element-wise. See an example below:

In python, it can be implemented by the code:

1
2
3
4
5
6
7
8


def addition(mat_a, mat_b):
 mat_c = []
 for row_a, row_b in zip(mat_a, mat_b):
 row_c = []
 for col_a, col_b in zip(row_a, row_b):
 row_c.append(col_a+col_b)
 mat_c.append(row_c)
 return mat_c

Multiplication in Matrix

There are two types of multiplication in matrices. Dot multiplication and element-wise multiplication. Element-wise multiplication is self-describing. We multiply each element of the two matrices just like addition/subtraction.

But dot multiplication is a little tricky. In dot multiplication, each element will be the sum of all the rows of the first matrix element-wise multiplied by all the columns of the second matrix one by one.

See the image below to understand both multiplications.

Shape of Matrix

The shape of a matrix is a feature that defines the structure of the matrix. The shape of a matrix is number of rows x number of columns. So the matrices used in the addition/subtraction section are of shape 3x4. In python, shapes are defined as tuples. So the shape of that matrix is (3, 4). We will understand this more in a bit when we start working with NumPy.

Tensor

Tensors are the core knowledge about data to use in Data Science. They are a more generalized form of Matrices. A tensor can be a single number or can be a more complex data structure than a matrix. According to Wikipedia:

In mathematics, a tensor is an algebraic object that describes a (multilinear) relationship between sets of algebraic objects related to a vector space.

Although this definition is a lot to digest, tensors aren’t actually that difficult to understand. The reason I introduced matrices before tensors are to make tensors more understandable. Think of tensors as blocks of data.

Let us start with a single number. This tensor has only 1 dimension (because it can be counted in only 1 way). And the shape will consist of only 1 integer (because of 1 dimension). How many values are there when counted in the first way? Only 1! So the shape will be (1,) in python tuple notation.

Now let’s increase the numbers count in the tensor. This tensor can also be counted in one way only (from left to right, or top to bottom if you write it in that way). So the dimension would be one. This first dimension is said to be dimension number 1. In this first dimension, there is a total of 5 values. So the shape of this tensor will be (5,).

Let’s increase those number count even more. But this time, we will increase the number in another direction. So a new dimension to the tensor will be added. And now the tensor can be counted in 2 directions. First, we can count the rows, and then count the columns (or vice-versa). So the dimension of this tensor will be 2. Typically, the rows are counted first, and then the columns. So the shape of the tensor will be (3, 5) because we will get 3 rows on the first dimension and 5 columns on the second dimension. Note that, this is exactly like a matrix. A matrix is nothing but a 2D tensor.

Finally, we can enlarge the tensor even more to a new dimension. This tensor will have 3 dimensions and the shape will be (3, 5, 3). First, we will have 3 planes when counting from left to right. Then in each plane, we have matrices of shape (5,3).

If you try to increase the dimension even more (create a 4D or 5D tensor), things will start to become difficult to imagine. But you can take 3 of the above 3D tensors side-by-side, and think of the whole picture as a 4D tensor.

Operations in tensor are the same as matrices discussed in the previous section. We do element-wise addition, subtraction, and multiplication. There is also a way to do dot products of two multidimensional tensors, but we are leaving this for later as it is too complex for the score of this series. Now let us see how to handle tensors in NumPy.

NumPy

Although NumPy calls tensors just arrays, they do all the operations among those like tensors. NumPy has a scalar type for a single value and ndarray type for tensors. To create an ndarray directly in code, you can create a list in python first, then call np.array(). Look at the below code:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19


import numpy as np

tensor_list = [
 [
 [1, 2, 3],
 [4, 5, 6],
 [7, 8, 9]
 ],
 [
 [11, 12, 13],
 [14, 15, 16],
 [17, 18, 19]
 ],
]

tensor = np.array(tensor_list)

print('Shape is :', tensor.shape) # (2,3,3)
print('Dimension is :', len(tensor.shape)) # 3

NumPy has a lot of convenient functions to create ndarrays. Look at the the code snippet below to understand some basic functions like these:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15


a = np.ones([2, 2, 3]) # A tensor of shape (2, 2, 3) with all ones
# Use the above function to create a tensor with given shape filled with any value
a2 = np.ones([2, 2, 3]) * 5 # A tensor of shape (2, 2, 3) with all 5s
a3 = np.full([2, 2, 3], 5) # Same and better than above

b = np.zeros([2, 2, 3]) # A tensor of shape (2, 2, 3) with all zeros

c = np.arange(10) # A tensor of shape (10,) populated with 0..9
d = c.reshape([2, 5]) # A tensor of shape (2, 5) populated with 0..9 (row first)

e = np.empty([2, 2, 3]) # This function is the fastest. Because it just captures the memory and does not initialize it. So there will be garbage values in this tensor

f = np.eye(4, 4) # Identity matrix with 1 in diagonal and 0s elsewhere. 

g = np.linspace(0, 10, 5) # A tensor of shape (5,) initialized with equidistant values from 0 to 10

Other than the functions above, you can also create ndarrays with the random submodule of NumPy. We will look at it soon.

Operations in NumPy

The most pleasing thing about NumPy is its operations’ simplicity. We will discuss some major kinds of operations here:

Arithmetic Operations

Numpy does operations on tensors just like python does operations on its primitive types. The only constraint here is that the arrays must be of the same shape. There is an exception to this which we will cover in the next section of broadcasting. Run the code below and see the example results yourself.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16


a = np.array([
 [1, 2, 3],
 [2, 3, 4]
])
b = np.array([
 [6, 7, 8],
 [7, 8, 9]
])

print(a + b) # Element-wise addition
print(a - b) # Element-wise subtraction
print(a * b) # Element-wise multiplication
print(a / b) # Element-wise division
print(a // b) # Element-wise integer division
print(np.dot(a, b)) # Matrix dot product. This will not work as shapes do to match
print(np.dot(a, b.T)) # Transpose b, then dot product

Tensor/Array Slicing

One of the most powerful features of NumPy is its tensor slicing. You can slice a tensor in any dimension very efficiently. It is just like list slicing in python. The only difference is that you need to indicate the slicing range in each dimension from the first. Think of these as lists that you are slicing in python. In the example below, take a closer look at the output shape of the arrays after slicing.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


a = np.array([
 [
 [1, 2, 3],
 [4, 5, 6],
 [7, 8, 9]
 ],
 [
 [11, 12, 13],
 [14, 15, 16],
 [17, 18, 19]
 ],
])

print(a) # (2, 3, 3)
print(a[0]) # (3, 3)
print(a[0][1]) # (3,)
print(a[0, 1]) # (3,) Same as above
print(a[:1, 1]) # (1, 3)
print(a[:1, :]) # (1, 3, 3)
print(a[:1, :, 2]) # (1, 3)

As a rule of thumb, always think of the slicing indices paired with their corresponding dimensions. The first index will cut the array in the first dimension. The second one will cut the array in the second dimension. If you don’t provide slicing indices in all the dimensions, NumPy will think that you are putting : for those. And if you want to take all the elements in an earlier dimension and slice in a later dimension, you can explicitly do that using : as I did in the last print statement of the example above.

Broadcasting in NumPy

This is the most difficult concept in this easy NumPy world. Broadcasting allows you to make operations among two tensors of different shapes. Whenever a dimension is found to be 1 in one of the ndarrays, NumPy automatically broadcasts that dimension to match the shape of two tensors. All you need to remember is the following two rules one of which must be satisfied for operations between two NumPy arrays:

The size of the corresponding dimensions must be the same.
The size of one of the corresponding dimensions is 1.

If the dimensions of two arrays are not the same, the dimension of the smaller array is padded on the left with ones to fulfill the requirements. Let us go through some examples in the image below:

A more confusion clearing demonstration is given below:

Pandas

Pandas is much similar to NumPy, but are much more focused on tabular data. Unlike ndarrays, it has Series (1D array) and DataFrame (2D array). Another advantage of using pandas is its manageability for large data. Pandas can easily manage 50,000 rows or more, and it can work on that data chunk by chunk. It also consumes larger memory than NumPy. The best place where you want to use pandas over numpy is where you need to work with CSV files or any other tabular data instead of multidimensional arrays.

We have already installed pandas library in the previous lesson in the newenv conda environment. Let us see how we can use it in our code. We will load a CSV file from the internet into memory with pandas.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14


import pandas as pd

df = pd.read_csv("https://datahub.io/machine-learning/iris/r/iris.csv")

#%% # This comment means you should start a new jupyter notebook cell from here

df.head(5) # Shows first 5 rows

#%%
df.tail(5) # Shows last 5 rows

#%%
# If you are not using an interactive jupyter notebook, you can use this to_string() method in terminal
print(df.head().to_string())

Pandas can be used to analyze, clean, explore, and manipulate data. The main two types in pandas are DataFrame and Series. A Series will always have a fixed datatype, but a dataframe can have many Series with different datatypes. We have already created a dataframe from a CSV URL. Let us see how we can create these two objects directly in code:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19


df = pd.DataFrame({
 'column 1': [1, 2, 3, 4],
 'column 2': ['box', 'car', 'juice', 'chair'],
})
df

#%%
ser = pd.Series([
 'a', 'b', 'c', 'd'
], index=['col 1', 'col 2', 'col 3', 'col 4'])
ser

#%%
# Accessing a column in dataframe will return a Series
df['column 1']

#%%
# Accessing a column in Series will return the actual value
ser['col 1']

This is all that is new about pandas to you! You can now to do all sorts of things that you could do with numpy. You can slice indices of a dataframe or Series or do operations among columns of two DataFrames.

1
2
3
4
5
6
7
8
9


# Accessing rows
df.loc[0] # First row
df.loc['column 1'] # row by name
df.loc[[0,1]] # First two rows
df.loc[:10] # First 10 rows

# Accessing columns
df.loc[:10]['columns 1'] # First 10 rows and column 1 as series
df.loc[:10]['columns 1'][1] # First 10 rows and column 1s second value

Besides reading CSV files with read_csv function, pandas can also read and write a dataframe into different types of files like JSON, CSV, xlsx, etc.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23


# json
df = pd.read_json('data.json')
df.to_json('data.json')

#%%
# csv
df = pd.read_csv('data.csv')
df.to_csv('data.csv')

#%%
# Excel
df = pd.read_excel('data.xlsx')
df.to_excel('data.xlsx')

#%%
# HTML
df = pd.read_html('data.html')
df.to_html('data.html')

#%%
# Pickle
df = pd.read_pickle('data.pickle')
df.to_pickle('data.pickle')

Finally, we will cover the most powerful method in pandas. The apply method. This method can be used to apply a function to any column or row in a dataframe. That function should receive a Series as an argument and return a value or Series. Depending on the value or series, a new Series or dataframe will be returned from apply after applying the function to that row or column. See the example below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21


import pandas as pd

df = pd.read_csv("https://datahub.io/machine-learning/iris/r/iris.csv")

#%%
def add_first_and_second_column(row):
 return row[0] + row[1]

df['col0 + col1'] = df.apply(all_first_and_second_column, axis=1)
df

#%%
# You can also use lambdas quickly
df['col0 + col1'] = df.apply(lambda row: row[0] + row[1], axis=1)
df

#%%
# This operation is just adding two columns, 
# So we can also do this without using apply
# But many operations WILL need to use apply
df['col0 + col1'] = df[0] + df[1]

Columns will be passed to the function when axis=0 and rows will be passed when axis=1. Finally, if your data is too large, you can get a brief description of it using the df.describe() method.

Conclusion

You are now a growing data scientist who knows his way around a dataset. You can now load data into memory, analyze and process the data in any way you want. Check out the documentation for both NumPy and Pandas. Then download a bunch of CSV and NumPy (np) files from online and play with them. In a later lesson, we will give you a guide on how to search and download a dataset more efficiently. Till then, keep patience and continue learning with me throughout this series.

Machine Learning Tutorial - Lesson 03

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Into the world of Machine Learning

Introduction

In this lesson, we will build our first Machine Learning algorithm with only Numpy. We will make you understand what machine learning really is. By this lesson, you will understand many types of classes and common keywords in Machine Learning.

Intuition

Let us first build a program that will try to guess a number you imagined by simply correcting the output of the program. The computer will try to guess a number and show it to you. Then you will tell the computer that the number is bigger or smaller than your guessed number. After that computer will guess again and show it to you. By iterating over this for long enough, you’ll see that the computer will slowly become accurate. Let us build the program step by step.

First, the computer will guess a number. We can easily do this using the random built-in module in python. Let us think that the number will be always between 0 and 100. We will also keep track of how many retries the computer had to do to guess my number correctly. It will start from 0.

1
2
3
4


import random

guess = random.randint(0, 100)
tries = 0

Now we will build the skeleton of the program. The computer will keep guessing until the guess is close enough. So we can do an infinite loop and ask the user when the guess is close enough to stop the loop. We also need to know whether the guess is bigger or smaller if it is not close enough.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30


import random

guess = random.randint(0, 100)
tries = 0

# This is known as the training loop in ML. Because we are updating our parameter "guess" in this loop
while True:
 tries += 1
 print("My guess is :", guess)
 print('''Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 :
 ''', end='')
 comp = int(input())
 # Input validation
 if comp not in [1, 2, 3]:
 print("Input is invalid")
 continue

 if comp == 1:
 print("I did it!!!")
 print("Total tries needed :", tries)
 break
 elif comp == 2:
 # Decrement guess
 elif comp == 3:
 # Increment guess

We only need the logic part of updating the guess to complete the program. We could simply increment/decrement guess by one. But this would be very tedious and the computer will need a lot of retries. So what we can do is increment the guess by a bigger value. Let’s call it jump (the guess is jumping towards the answer).

1
2
3
4
5
6
7


jump = 20

# in decrement part
guess -= jump

# in increment part
guess += jump

But if we keep the jump fixed, do you see what went wrong during the play? The guess never comes close to the actual answer. So we somehow need to decrement the value of jump throughout the iterations. But what if we need to jump only in a positive direction to get to the result. In that case, no need to decrease the jump length as it will only slow the convergence process.

So what can we do here? The solution is that we keep track of the jump direction. If the jump direction changes, we know that we accidentally jumped too far. So we need to jump in the other direction a little smaller. Let’s do it in the code.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39


import random

guess = random.randint(0, 100)
tries = 0
jump = 20
last_comp = 0 # Keeping track of the direction
decrement_jump = 2

while True:
 # 80
 tries += 1
 print("My guess is :", guess)
 print('''

 Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : ''', end='')
 comp = int(input())
 if comp not in [1, 2, 3]:
 print("Input is invalid")
 continue

 # Oops, we jumped too far. So decrement jump
 if last_comp != comp:
 jump /= decrement_jump

 if comp == 1:
 print("I did it!!!")
 print("Total tries needed :", tries)
 break
 elif comp == 2:
 guess -= jump
 elif comp == 3:
 guess += jump

 last_comp = comp

The program is now complete. Run it and the computer should be able to guess your answer after some retries like below where I guessed 20:

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142


My guess is: 99
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 89.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 79.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 69.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 59.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 49.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 39.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 29.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 19.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 3
My guess is : 24.0
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 21.5
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 3
My guess is : 22.75
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 22.125
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 21.5
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 20.875
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 20.25
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 19.625
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 3
My guess is : 19.9375
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 3
My guess is : 20.25
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 2
My guess is : 20.09375
Is the guess close enough?
 1: Yes!!
 2: The guess is bigger
 3: The guess is smaller

 : 1
I did it!!!
Total tries needed : 20

Understanding Machine Learning

Do you know that you just built a complete machine learning algorithm? Yes, you did! Machine Learning keeps an internal state of the program and tries to update it to get to a better answer if you give it some data to learn. Let us look through the analogy of this program with the concepts of machine learning.

1. Internal State/ Weight: guess variable itself is the internal state that gets updated when you provide new data for the program to learn. All machine learning algorithms must have some internal states to maintain during the learning and prediction process. Some deep learning models even have gigabytes of weights to learn from examples. We will learn about weights in a little bit.

2. Loss: Although we do not directly have a loss value or loss function in the above program, jump can be thought of as a loss value. The farther we are from the answer, the bigger the loss. We can see that the loss (jump) is decreasing slowly. That means the algorithm is getting better to predict my guess. All Machine Learning algorithm’s main objective is to decrement/optimize the loss as much as possible.

3. Learning rate: The learning rate is the parameter that defines how fast the ML algorithm learns. It must not be too high or too low. In our program, decrement_jump can be thought of as the learning rate. If the learning rate is too high, we will never converge and oscillate around the minima. And if the learning rate is too low, it will take a lot of time for the algorithm to converge.

4. Optimizer/Optimizing algorithm: So a machine learning algorithm has data, weights, loss, and its learning rate. Previously, I said that the main objective of the ML algorithm is to optimize the loss function. All we need to do is update our weights somehow with the help of loss and learning rate. This is the job of an optimizer. In our program above, the incrementing and decrementing part inside conditionals and the logic behind last_comp to find a change of direction of jump is the optimizing algorithm. We will learn about different optimizers and gradient descent algorithms soon.

5. Convergence threshold: Our program has to meet criteria to stop the while loop. Those criteria are when we think computers guessed close enough. This is known as convergence criteria. Most of the time a value is preset, so when the loss gets smaller than that value, convergence criteria are met and the ML algorithm stops learning. That value is known as the convergence threshold.

6. Learning iteration: At every iteration, we are providing data (guess is bigger or smaller) or a batch of data to the model. The model then calculates the loss with the given data and weights. Each time the algorithm is updating the weights, one learning iteration is passed.

7. Epoch: If we provide a big dataset to an algorithm, the algorithm has to go over the data several times to learn better. When the whole data is provided beforehand, it is known as offline learning. In offline learning, when the ML algorithm goes over the whole data once, one epoch is passed.

8. Accuracy: After training a model like our program above, we have to test the model to know how good the algorithm is. We can do this by determining the accuracy of the model. Accuracy can be calculated differently based on different objective functions. In the case of our guessing game program, the accuracy would be (answer-guess)/answer x 100%. Most of the time, accuracy is calculated with percentage.

9. Activation function: This function is not available in our guessing game program. But the job of the activation function is to post-processed the output of ML algorithms to keep it in an appropriate shape. Some ML algorithms sometimes try to make the output too big, too small, negative, or any undesirable value. At those times, activation functions help to keep the output within a boundary.

The Data

Data are the most important thing in machine learning. Because all machine learning algorithms task is to learn that data. Machine learning algorithms are first learned by some data, and then the model is used to predict the unknown data. In this sense, data is divided into two parts. A lot of resources found online sometimes mislead the reader about train, test, and cross-validation data. There are even some places that mislead the reader by saying that we validate our model with the test data, which is completely wrong. So bear with me, after this lesson, you will have a solid idea of how data is divided and used in ML algorithms.

Let us take an example of vehicle data, where our objective is to estimate the price of a car from the features of that car.

1. Training Data: This is the data that we know everything about. We know what should be the output of this data. So we can calculate the loss of our model from this data. We can also calculate the accuracy of the model from this training data. This is the data that is used in the training loop of an ML algorithm. In the example above, all the features along with the prices are the training data.

2. Test Data: This is the data that we need to understand using our ML algorithms. We need to predict the result of unknown data with our model. Suppose you learned a model from the given training data of vehicles. Now if you have the feature lists of a set of cars, and you do not know their price, that will be your test data.

So if you train on the whole training data, you will not have any solid way to test the accuracy of your model. Maybe your model will work on seen data but will break on unseen data. You won’t be able to get the accuracy from your test data because you do not know the prices of vehicles of the test data. To solve this, the training data is again divided into two parts.

1. Actual Train Data: This data is the actual training data that will be used in the training loop. In most cases, this is 80% or 75% of the whole train data.

2. Cross-Validation Data: This is kept separate from the actual training data and is not used in the training loop. So after training, this cross-validation data is unseen to the model. So we can cross validate our model on both seen and unseen data from the train data. Most of the time, this data is 20% or 25% of the training data.

Dataset attributes have a classification according to their usage:

1. Feature: These are the attributes or columns that the machine learning model will analyze and learn. In the vehicle dataset, all the columns except price are features.

2. Label: These are the attributes or columns that the machine learning model will try to predict. This label will be used to calculate loss and accuracy. In the vehicle dataset, the column “price” is the label.

To illustrate the whole scenario with the vehicle dataset example, look at the image above. The prices in green are the label of the data, and the rest of the columns are features. We will understand more about different types of features like numeric, categorical features, etc soon.

Different classes of Machine Learning

There are many kinds of machine learning algorithms depending on the objective and provided data. We will discuss some of them below:

Supervised Learning and Unsupervised Learning

These classes of learning are the most popular among all. If the provided data has a label on it, then it is supervised learning. If the process or algorithm does not need any label, then it is unsupervised learning.

The above model for the vehicle dataset is supervised learning if you want to predict the price or any feature from the rest of the attributes. The same data can be used for unsupervised learning if you want to cluster similar vehicles or detect outlier vehicles.

Usually, unsupervised learning is a little harder than supervised learning. Because it is difficult to determine the loss or goodness of a model as there is no label to compare with. For clustering a dataset, you can use the distance between the mean of two clusters or inverse of distance among points in similar clusters as goodness of the model. We will first encounter many supervised learning methods. Later on in this series, we will go through data clustering, outlier detection, data augmentation, synthesis, and many other unsupervised learning applications.

Instance-based Learning and Model-based Learning

Machine learning can be divided into two categories based on the learning process and weights management.

Think of the example ML program you implemented at the beginning of this lesson. Is the weight guess saved somewhere for future use? Can the model work better for a later case when you guess a different number? No. This is what instance-based learning is. In instance-based learning, the algorithm works for only the current instance of the data. It is highly dependent on the data and does not work later if you do not provide data.

On the other hand, some ML algorithms can save their weights for later use. These kinds of algorithms can relearn and improve themselves if more data is provided. When doing prediction, it does not depend on the data. All the deep learning algorithms are model-based learning processes.

Instance-based learning algorithms are usually faster than model-based learning. For example, the Bayesian network model or KNN clustering algorithm is always faster than a deep learning algorithm when learning.

Offline/Batch Learning and Online Learning

Depending on the data again, ML algorithms can be divided into two kinds. Machine learning algorithms can learn while the data is given live, or learn after getting the complete dataset provided at once.

In the above number guessing program, the data is given to the computer one by one. The data looked like the following at each step:

1
2
3
4
5
6


Step 1: 99 is bigger than guess
Step 2: 89 is bigger than guess
Step 3: 79 is bigger than guess
Step 4: 69 is bigger than guess
Step 5: 59 is bigger than guess
...

This type of learning algorithm is known as the Online learning process. The data is given live while the training loop is running.

On the other hand, if you try to do a dog-cat classification, many images of dogs and cats are given to the learning model at once. The model runs for several epochs for all the images and then gets ready to predict a test image. This kind of learning is Offline/Batch learning. Because we are providing a batch of dataset at once.

Conclusion

This lesson is very important to understand for your Machine learning career. This is the fundamental of machine learning. We will go through some core mathematics behind ML in the next lesson. I would recommend you go through the algorithm multiple times, print different variables in the training loop, and understand the whole process.

Machine Learning Tutorial - Lesson 04

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Hands-on first Machine Learning Algorithm from Scratch

Introduction

Previously, we have created a program, that behaves almost like a machine learning algorithm. But in this section, we will officially create our first machine learning program. We are going to implement linear regression.

Objective

Our objective is to predict house prices based on several features of a house. As there is more than one feature for this dataset, the task is a multivariate linear regression.

Go ahead and grab the Kaggle competition dataset from the website. We will only need the train.csv file from the dataset.

Reading the data

Let us first start with the data acquisition and preprocessing. We grab the data just like we did in the last lesson.

1
2
3
4
5
6


import pandas as np
import numpy as np

#%%
df = pd.read_csv('train.csv')
df.head()

Data shuffling

In many cases of data acquisition, the data seems to be written to a CSV file in an orderly manner. For example, the data can be there in order of SalesPrice or any other feature. But in this house price prediction case, there is no actual meaning to the order of the houses in the CSV file. We do not want our ML algorithm to pick this order and learn something from it. So it is always a good idea to shuffle the data first if there is no meaning in the ordering of the data.

Shuffling can be done in many ways with many libraries. As we only have introduced pandas and NumPy, we will shuffle the CSV directly through pandas sample method. Later you will know that it is even easier to shuffle with just shuffle(data) using scikit-learn.

1

df = df.sample(frac=1).reset_index(drop=True)

We are sampling the data (all the data because we put fraction=1) which is the same as shuffling. When sampled, pandas try to keep the index of the shuffled data the same as the original one. This will create a problem if we want to access data using Loc. We can reset the index with the reset_index method.

Preparing features and Labels

After shuffling the data, we preprocess the data to directly feed to an algorithm pipeline. The only numerical features we are going to use are OverallQual, GrLivArea, and GarageCars. The label will be SalesPrice. So we will take only these 3 columns into the input data X and 1 column to label data y.

1
2


x = df[['OverallQual', 'GrLivArea', 'GarageCars']].to_numpy()
y = df['SalePrice'].to_numpy()

Input feature normalization

You can notice that the input feature x has different ranges for different columns. We will later use a gradient descent algorithm for optimizing the loss. You will start to understand in a moment, but the gradient descent algorithm works badly when different data are in different ranges. We can solve this by normalization.

The equation of normalization is as follow:

1

X_new = (X - mean(X_old)) / standard_deviation(X_old) ; X is a column representing a feature of all samples

We will also add a column of 1s to the feature set. We will explain the reason in a bit when discussing the prediction process.

1
2


x = (x - x.mean(axis=0)) / x.std(axis=0)
x = np.c_[np.ones(x.shape[0]), x]

We could also do min-max feature scaling if we wanted. The result will be almost similar.

After processing, the data will look something like this:

Splitting the data to Train and Test

As the data is already shuffled, we can just take the first 90% of the data for training, and 10% of the data for validation. This can be done with NumPy’s slicing operation.

1
2
3
4
5
6
7


X_train = x[:int(len(x)*0.9)]
X_test = x[int(len(x)*0.9):]

y_train = y[:int(len(y)*0.9)]
y_test = y[int(len(y)*0.9):]

X_train.shape, y_train.shape, X_test.shape, y_test.shape

Gradient Descent Algorithm

Nowadays, gradient descent algorithms remain pretty much the same for all the algorithms from stochastic linear regression, to advanced deep learning. This algorithm needs to have a loss function (loss_i), and a learning rate parameter. All this algorithm does is subtract the gradient of loss of all samples from the original weight at each learning iteration. The actual derivation of the algorithm requires a good knowledge of calculus and is left for self-learning. The updating speed of the weight can be controlled using the learning rate (alpha) parameter (just like the decrement_jump in the guessing game).

Let us see the equation to update weight directly from Wikipedia:

Here, the parameter that needs to be updated is a. At every learning iteration, it will be updated by the gradient of loss. $x_n$ is the $n$th sample and $F(x_n)$ is the loss of that sample. The $\Delta F(x_n)$ defines the gradient of loss. This is actually the differentiation of loss with respect to the input. Without burning our brains too much, the calculated result of the derivative of loss (the gradient) is half of the dot product of input matrices transpose with the loss.

The simplified equation goes something like this.

$$
weight_{n+1} = weight_n - lr * 0.5 * X^T.loss
$$

This updating process will happen a fixed number of times, which is known as the number of epochs. We can keep track of the losses during the training loop in an array cost_history.

1
2
3
4
5
6
7
8


def gradient_descent(X, y, w, lr, epochs):
 cost_history = np.zeros(epochs)
 for i in range(epochs):
 errors = predict(X, w) - y
 grad = (1 / len(X)) * X.T.dot(errors) # mse can also be calculated with matrix dot product
 w = w - lr * grad
 cost_history[i] = loss(X, y, w)
 return w, cost_history

Prediction and Loss Function

Let us think of the loss function as a subtraction between the prediction and label. This is for only one sample of data. But for a batch of sample data, we need to normalize the loss. We can sum all the individual losses and divide them by the number of samples. It is not always necessary, but for some derivation explanatory reasons, the loss is further divided by 2.

1
2
3
4


def loss(X, y, w):
 errors = predict(X, w) - y
 pred_loss = 1/(2 * len(X)) * errors.T.dot(errors)
 return pred_loss

The prediction or output will be the multiplication of the weights and the input feature set. This came from the equation $y = w * x + b$. Here, $x$ is the input features, and $w$ is the weights that gradient descent will learn. Remember that we added a column of 1s while preprocessing the data? That $1$ is called bias, $b$ in the equation. We can implement this on python pretty easily because we already prepared our data to be compatible with bias.

1
2


def predict(X, w):
 return X.dot(w)

The Training Loop

All the functions are now defined properly. The starting of the program will be the initialization of all the parameters. There are two types of parameters in ML algorithms.

1. Training Parameters: These are the parameters that are updated in the training loop. These are also simply called parameters. This can be only a single floating value or a tensor of size more than gigabytes.

2. Hyperparameter: These are the parameters of the parameter updating process. These are set by us before running the algorithm. Examples are learning rate, number of epochs, parameter initialization distribution, etc.

1
2
3
4
5
6


# Initialize parameters
w = np.zeros(4)

# Initialize hyperparameter
epochs = 800000;
lr = 0.15;

Then we call the gradient descent function and the model will be trained.

1
2
3


w, cost_history = gradient_descent(x, y, w, lr, epochs)
print('Initial loss =', cost_history[:5].mean())
print('Loss after training =', cost_history[-5 :].mean())

We can clearly see that the loss is decreased. So our model has learned something meaningful from the feature set.

Conclusion

In this lesson, you have officially created the first well-established machine learning algorithm. You became familiar with the math behind the most popular machine learning algorithms, gradient descent. I will recommend you to go through the whole algorithm, print variables at different lines in the python code to understand how it is actually working. In most of the applications in machine learning, you won’t be using this type of code that much; thanks to many modern python modules that implement this algorithm for us. Finally, as an exercise, you can grab any other dataset online and try to preprocess and apply this algorithm to predict something new. Don’t forget to share your experience and problems with us about your new implementation in the discussion section.

Machine Learning Tutorial - Lesson 05

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Understanding the Data: Statistics

Introduction

In this lesson, we will get started with some very basic statistical analysis. A statistic of data is just a summary of that data for humans and computers. Imagine you have a medical history of 2 million people from multiple hospitals, and you want to know which disease is worst among a set of diseases. Of course, you cannot go through all the 2 million people and keep comparing by yourself. With the help of statistics and good computational power, you can do it within hours. You can visualize the body temperature box plots and see that one disease causes more fever than the others.

Concepts of Statistics in Machine Learning

Statistics is a vast field of study very close to machine learning. The core mathematics of machine learning is all about statistics. So if you want to learn theoretical machine learning, you should have advanced statistics knowledge. We will walk you through some basic statistical knowledge that you will need for applied machine learning.

Probability theorem

In Mathematics, probability is the likelihood of an event. The probability of an event going to happen is 1 and for an impossible event is 0. In a domain or problem set, the sum of the probability of all the events that might happen is 1. And the probability of an event can never be negative.

Most of the time, we say that there is a 50-50 chance of raining today. What we mean is the probability of raining today is 0.5. And the probability of not raining is 0.5. In the event space, there are only two events that might happen in this domain. raining and not raining. Note that the sum of the probability of these two events is (0.5 + 0.5) = 1.

Now think of a rolling die. There are 6 faces of a die. So there could be 6 events of a rolling die. Thus event space is $E = {1, 2, 3, 4, 5, 6}$. If the die is normal, then each of these events will have an equal probability to occur. And the sum of the probability of events will always be 1. So each event will have a

$$\frac{total\ probability}{#\ of\ events} = 1/6$$

Understanding the Venn Diagram helps a lot to understand probability. For the two domains discussed above, the Venn diagrams are shown below:

All of these events were mutually exclusive. This means only one event can happen at a time. Of course, you cannot have a rainy and no-raining state at the same time. Or you cannot get a 1 and 6 from a single die roll. There can be mutually inclusive events as well.

Think of guessing any number. That number can be a prime number (or not). That can also be a number less than 10 (or not). But can it be both prime and below 10? Yes, it can. So if we take events $E1 = {x: x < 10\ &\ x \in R}$ and $E2 = {x: x\ is\ prime}$, then $E1$ and $E2$ are mutually inclusive.

For mutually exclusive events $A$ and $B$, the probability of happening both events is zero.

$$P(A \cap B) = 0$$

But for mutually inclusive events, both of them can happen, so there is a probability of both events happening.

When we want to know the probability of two events happening, then we can use this formula:

$$P(A \cup B) = P(A) + P(B) - P(A \cap B)$$

Also, the probability of something not happening is the inverse of the probability of something happening.

$$P(A’) = 1 - P(A)$$

Probability Mass function and Density function

Probability mass function and density function are two functions in the world of probability calculus. Both of these are functions for a random variable. A random variable is a variable whose value is unknown or a function that assigns values to each of an experiment’s outcomes.

Probability Mass function is defined for discrete values. For example, a die will always have ${1, 2, 3, 4, 5, 6}$ in the event space. It won’t contain any value like 1.5. Probability Mass Function (PMF) is evaluated at a value corresponding to the probability that a random variable takes that value. It is just a fancy way of describing the same probability theory we discussed above. For a die,

$$P(1) = \frac{1}{6}$$

$$P(2) = \frac{1}{6}$$

$$P(3) = \frac{1}{6}$$

$$P(4) = \frac{1}{6}$$

$$P(5) = \frac{1}{6}$$

$$P(6) = \frac{1}{6}$$

The two probability theorem must be satisfied by a PMF. So,

$$ \sum{P(X)} = \frac{1}{6} +\frac{1}{6} +\frac{1}{6} +\frac{1}{6} +\frac{1}{6} +\frac{1}{6} = 1 $$

and

$$ P(X) >= 0 $$

Probability Density Function (PDF) is the same as PMF, but for a continuous random variable. So for continuous, we will integrate the random variable instead of summing.

$$ \int{P(X)} = 1 $$

Different kinds of Data Distribution

Gaussian Distribution

According to Wikipedia:

In probability theory, a normal (or Gaussian or Gauss or Laplace–Gauss) distribution is a type of continuous probability distribution for a real-valued random variable. The general form of its probability density function is

$$f(x) = \frac{1}{\sigma \sqrt{2\pi}} e^{-\frac{1}{2}(\frac{x-\mu}{\sigma})^2}$$

In simple words, it is a distribution of data that is centered around a fixed value with no bias in any direction (left or right). The below image is a gaussian distribution. Gaussian distribution is also known as Normal Distribution.

Gaussian distribution has two important properties. The mean of the data, and the variance or standard deviation of the data. In the above image, you can see that mean can change the mode of the data. And standard deviation can change the spreadness or skewness of the data.

Here is the implementation of random Gaussian distribution in python and NumPy:

1
2
3
4
5
6
7
8


import numpy as np

n = 1000
mean = 0.0
variance = 1.0

x = np.arange(n)
fx = np.random.normal(loc=mean, scale=variance, size=n)

Uniform Distribution

When you roll a fair die, the outcomes are 1 to 6. The probabilities of getting these outcomes are equally likely and that is the basis of a uniform distribution. The formula of Uniform distribution is given below, where the range of data $x$ is $[a,b]$.

$$f(x) = \frac{1}{b-a}$$

Here is the implementation of random Uniform distribution in python and NumPy:

1
2
3
4
5
6


n = 1000
a = 5.0
b = 10.0

x = np.arange(n)
fx = np.random.uniform(low=a, high=b, size=n)

Poisson Distribution

Poisson Distribution is the closest distribution that represents various events in real life. If you try to model a queueing process in a ticket counter, it will follow a Poisson Distribution. Poisson Distribution is applicable in situations where events occur at random points of time and space wherein our interest lies only in the number of occurrences of the event.

The main conditions of data in Poisson Distribution are:

Any event should not influence the outcome of another event.
The probability of an event over a short interval must equal the probability of that event over a longer interval.
The probability of an event in an interval approaches zero as the interval becomes smaller.

The parameters in Poisson Distribution are $\lambda$ and $t$. Here, $\lambda$ is the rate of an event happening, and $t$ is length of time interval. The formula is as follows:

$$P(x) = e^{-\mu \frac{\mu^x}{x!}}$$

The plot of this distribution is very similar to the plot of normal distribution.

Here is the implementation of random Poisson distribution in python and NumPy:

1
2
3
4
5


n = 1000
interval = 1.0

x = np.arange(n)
fx = np.random.poisson(lam=interval, size=n)

Exponential Distribution

The exponential distribution is widely used for survival analysis. To get an understanding of the expected life of an object, we can use exponential distribution. The formula is as follows:

$$f(x) = \lambda e^{-\lambda x}$$

Here, the parameter $\lambda$ is known as the failure rate of an object at any time $t$, given that it has survived up to time $t$.

So in common sense, an object will survive less if we consider it for a longer time. That is why exponential distribution drops rapidly to zero when x (or $t$) is increased.

Here is the implementation of random Poisson distribution in python and NumPy:

1
2
3
4
5


n = 1000
lamda = 1.0

x = np.arange(n)
fx = np.random.exponential(scale=lamda, size=n)

Statistics on Data Preprocessing

Data normalization

Data normalization is one of the most important steps in data preprocessing (Processing data before any kind of inference). Normalization is a scaling technique in which values are shifted and rescaled so that they end up being inside a range. It is also known as Min-Max scaling.

If we want to normalize a dataset $X$ in the range of [0, 1], we can do this by the following formula.

$$X’ = \frac{X - X_{min}}{X_{max} - X_{min}}$$

If we want the range to be something else, we can do this with a little more tweaking of the above formula.

$$X’ = \frac{X - X_{min}}{X_{max} - X_{min}} * (X_{newmax} - X_{newmin}) + X_{newmin}$$

In python, you can normalize the data X by the snippet below:

1

X_normalized = (X - X.min())/(X.max() - X.min()) * (new_max - new_min) + new_min

Data standardization

Data standardization is another kind of preprocessing very similar to Normalization. Here, the values are centered around the mean with a unit standard deviation. This means that the mean of the attribute becomes zero and the resultant distribution has a unit standard deviation. The formula is:

$$X’ = \frac{X - \mu}{\sigma}$$

Where $\mu$ is the mean of that attribute, and $\sigma$ is the standard deviation of that attribute. We can compute both of them like below:

$$\mu = \frac{\sum_{i=0}^N{X_i}}{N}$$

$$\sigma = \sqrt{\frac{\sum{(x_i - \mu)^2}}{N}}$$

So, where can I use normalization and standardization? Below is a rule of thumb:

When the data does not follow a Gaussian distribution, you should use normalization feature scaling. Typically, the K-Nearest neighbor and Neural Network Algorithm need feature scaling.
When the data follows a Gaussian distribution, then you should use standardization instead of normalization. Remember, standardization does not have any bounding range. So this might even break your inference model if used inappropriately.

We already did standardize our data in one of our previous lessons. Here is the same code snippet again.

1

X = (X - X.mean(axis=0)) / X.std(axis=0)

Correlation between Variables

In many cases of different machine learning algorithms, you might encounter some highly correlated features. Here, correlated means, one feature can describe another feature very closely. If that is the case, most ML algorithms get biased toward that feature. Also, those duplicate features are extra, meaning they can be removed from the learning process without losing much of the information in the data. Removing highly correlated features fastens the time learning by an algorithm.

A correlation could be positive, meaning both variables move in the same direction, or negative, meaning that when one variable’s value increases, the other variables’ values decrease. As stated earlier, the performance of some algorithms can deteriorate if two or more variables are tightly related. This phenomenon is called multicollinearity.

Correlation between two features can be detected using the Pearson correlation coefficient. The formula is:

$$\rho (X_1,X_2) = \frac{cov(X_1, X_2)}{\sigma_{X_1}\sigma_{X_2}}$$

If $\rho$ is positive and close to 1, the features are highly positively correlated. And if it is close to -1, then they are highly negatively correlated. Otherwise, there is no correlation between the features.

In python, we can calculate correlation between two data as follows:

1
2
3


X_1 = np.array([-2, -1, 0, 1, 2])
X_2 = np.array([4, 1, 3, 2, 0])
my_rho = np.corrcoef(X_1, X_2)

Statistical Hypothesis

Hypothesis testing is a statistical method that is used in making statistical decisions using experimental data. Hypothesis Testing is an assumption that we make about the population parameter.

For example, if you analyze a hospital and try to test whether males are affected more by COVID-19 than women, then it is called hypothesis testing.

To do this testing, we need to have a hypothesis known as $H_0$ (the null hypothesis). The basic hypothesis is normalization and standard normalization so try to understand these two topics first from above.

The alternative hypothesis is the hypothesis used in hypothesis testing that is contrary to the null hypothesis. It is usually taken to be that the observations are the result of a real effect (with some amount of chance variation superposed).

After fixing the null and alternative hypothesis, we select a level of significance. The level of significance is how sure you are when you are trying to select a decision.

With a level of significance, we detect a P-Value (calculated probability), probability of finding the observed, or more extreme, results when the null hypothesis ($H_0$) of a study question is true. If we get a very small p-value (0, 0.001, 0.005), we reject the null hypothesis.

Hypothesis testing can be done with different kinds of statistical testing. We will discuss t-test in this lesson. The equation of t-test is given below:

$$t = \frac{\mu_{X_1} - \mu_{X_2}}{\sqrt{\frac{S_{X_1}^2}{n_{X_1} }+ \frac{S_{X_2}^2}{n_{X_2}}}}$$

Where $S$ is the standard deviation of the sample. And $\mu$ is the mean.

We can implement the hypothesis testing easily in Python. But we are going to need a table to compare and get the p-value.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33


import numpy as np
from scipy import stats # dont worry about this. We cover it later

N = 10
# Two gaussian distributions
X_1 = np.random.randn(N) + 10 # mu = 10, var = 1
X_2 = np.random.randn(N) + 3 # mu = 3, var = 1

X_1_var = X_1.var(ddof=1)
X_2_var = X_2.var(ddof=1)

S = np.sqrt((var_a + var_b)/2)

# Calculate the t-statistics
t = (X_1.mean() - X_2.mean())/(S*np.sqrt(2/N))

# Now compare the value of t from a t-statistic table to get the p-value. 
## This can be done with a higher-level library scipy. We will learn about this library in a lesson soon.

# Degrees of freedom
df = 2*N - 2
#p-value after comparison with the t 
p = 1 - stats.t.cdf(t,df=df)

# Now that we have a p-value

print(f"t = {t}")
print(f"p = {2*p}")

## Cross Checking with the internal scipy function
t2, p2 = stats.ttest_ind(X_1,X_2)
print(f"t2 = {t2}")
print(f"p2 = {p2}")

Conclusion

In the next lesson, you will get started with beautiful data visualization. In that lesson, we will plot each distribution with some dummy parameters. We can also interact with distributions and understand how they are working.

Machine Learning Tutorial - Lesson 06

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Data Visualization

Introduction

In this lesson, we will introduce two new giant libraries: matplotlib and plotly. There are tons of visualization libraries in python, but matplotlib is the lowest level and most of the other libraries (e.g. seaborn, ggplot, bokeh, etc.) are built only on matplotlib. We are also introducing you to plotly to make you understand what Interactive plots mean. Without any delay, let us get started.

Matplotlib

Let us first install the library matplotlib. We can do it with both pip or conda.

1
2


pip install matplotlib
conda install matplotlib

You can follow the official documentation for matplotlib. But it is quite large, so we are here to summarize everything within only half of this lesson. Also, matplotlib has two kinds of API. The functional and Object API. We will only work with Object API as it is more modern than the other.

Matplotlib graphs your data on Figures (i.e., windows, Jupyter widgets, etc.), each of which can contain one or more Axes (i.e., an area where points can be specified in terms of x-y coordinates, or theta-r in a polar plot, or x-y-z in a 3D plot, etc.). The simplest way of creating a figure with an axes is using pyplot.subplots. We can then use Axes.plot to draw some data on the axes:

1
2
3
4
5
6
7
8


# Matplotlib has a lot of submodules to work with. 
# But in general cases, all we need is the pyplot submodule. 
# By convention, it is always imported as plt
from matplotlib import pyplot as plt

fig, ax = plt.subplots() # Create a figure containing a single axes.
ax.plot([1, 2, 3, 4], [1, 4, 2, 3]) # Plot some data on the axes.
fig.show()

Now let us understand each part of a full-fledged figure in matplotlib. See the image below:

Matplotlib works best with only NumPy arrays. So if you have something else like a NumPy matrix or pandas dataframe, it is best to convert them to NumPy first. Pandas has a df.values attribute and NumPy has np.asarray(matrix) method to do this.

We only need to work with three types of objects in matplotlib. Try to understand these by seeing the picture below:

So we need to create a figure. Inside that figure, we create several axes. Inside those axes, there will be 2 or 3 axis (like real-life x-axis and y-axis). All of these are done below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25


x = np.linspace(0, 2, 100)

fig, all_axes = plt.subplots(
 2,3, # The subplots will be in a grid of size 2x3 
 figsize=(20, 10), # The figure size is 20*80=1600 pixels width and 10*80=800 pixels height 
 dpi=80 # Define the density per pixel (number of pixels in 1 inch)
) # Create a figure and 6 axes

((axs11, axs12, axs13),(axs21, axs22, axs23)) = all_axes

axs11.plot(x, x**1, label='linear') # Plot some data on the axes.
axs12.plot(x, x**2, label='quadratic') # Plot more data on the axes...
axs13.plot(x, x**3, label='cubic') # ... and some more.

axs21.plot(x, x**1, label='linear') # ... and some more.
axs22.plot(x, x**(1/2), label='quadratic') # ... and some more.
axs23.plot(x, x**(1/3), label='cubic') # ... and some more.

for axs in all_axes.reshape(-1): # Loop through all axes. Here axes are np.array of shape (2,3). We reshape it to 1D.
 axs.set_xlabel('x label') # Add an x-label to the axes.
 axs.set_ylabel('y label') # Add a y-label to the axes.
 axs.set_title("Simple Plot") # Add a title to the axes.
 axs.legend() # Add a legend.

fig.show()

We will go through each type of plot one by one right after we introduce an interactive plot next.

Plotly

Plotly is a plotting/data visualization library written in Javascript. It is ported to python and used with HTML, JS right inside python. The most amazing feature of plotly is that it helps you create interactive plots. Interactive plots are those plots where you can interact. You can zoom in, filter a subset of the data visualized, and change the scale of any axis. All of these and much more are achievable inside plotly.

The only downside of plotly is its performance. It is reasonably very slow compared to matplotlib as it is manipulating a full-fledged website written with HTML, CSS, and Javascript.

Installing plotly is a little trickier than installing other python libraries. First, install the plotly core library using pip or conda.

1
2
3
4
5


pip install plotly
# or
# The latest version of plotly is not available in the base channel. 
# So we install it from its own channel
conda install plotly -c plotly

This will install everything required to use plotly. But, if you want to install it into jupyter notebook or jupyter-lab, then you need to install ipywidgets python package.

1
2
3


pip install ipywidgets
# or
conda install ipywidgets

For jupyter-lab, you want to add the jupyterlab extension as well.

1
2


# Only for jupyter-lab
jupyter labextension install jupyterlab-plotly

Mostly, plotly works with Graph Objects and traces inside them. On the other hand, plotly provides a submodule named express. This is a layer above plotly to make common types of plotting much easier. For this lesson, we will only use plotly express for most of the plotting.

Unlike matplotlib, plotly works best with pandas Dataframes. So it is best to create a dataframe from the NumPy array or matrix to visualize with plotly.

Plotly also has a set of datasets inside the library mostly for demo purposes. Let’s use the iris dataset and plot it with plotly express.

1
2
3
4


import plotly.express as px
df = px.data.iris()
fig = px.scatter(df, x="sepal_width", y="sepal_length", color="species")
fig.show()

Let us start to draw different kinds of plots for different kinds of data using both matplotlib and plotly library.

Types of Plots

There are common types of plots that are well established for different classes of data. Almost all the plotting libraries including matplotlib and plotly have implementations of these types of plots. Let’s discuss that one at a time.

Scatter Plot

Given two continuous features, we can create a plot where the x-axis represents one and the y-axis represents another. Then each of the samples will represent a point on the plot. This is ideal when we want to detect any relational pattern between two features.

Let’s implement this on matplotlib:

1
2
3
4
5
6


import matplotlib.pyplot as plt

x = np.random.randint(0, 100, 15)
y = np.random.randint(0, 100, 15)
plt.scatter(x, y)
plt.show()

The same thing for plotly is:

1
2
3
4
5
6


import plotly.express as px

x = np.random.randint(0, 100, 15)
y = np.random.randint(0, 100, 15)
fig = px.scatter(x=x, y=y)
fig.show()

For the conciseness of the lesson, we will now only write the data generation and plotting part in a single code snippet.

Line Plot

Line plots are very likely to Scatter plots. The main difference in line plots is that the domain of the x-axis will be continuous. Usually, line plots are best to see any kind of function. Also, we can easily detect the trend (upward or downward) in line plots.

Moreover, if you plot multiple functions in the same line plot, you can compare them pretty easily. In later lessons, we will use line plots to see the loss functions, accuracy, etc. with respect to epochs or iterations.

Let us create the same line plot in both matplotlib and plotly.

1
2
3


x = np.arange(100)
plt.plot(x, np.sin(x)) # matplotlib.pyplot
px.line(x=x, y=np.sin(x)) # plotly.express

Bar Charts

The bar graphs are used in data comparison where we can measure the changes over a period of time. It can be represented horizontally or vertically. The longer the bar it has the greater the value it contains.

In later lessons, we will use bar charts to see different kinds of categorical variables and compare balance in classes.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


fruits = ['Apple', 'Orange', 'Pineapple', 'JackFruit', 'Banana']
amount = [23,17,35,40,12]

fig = plt.figure()
ax = fig.add_axes([0,0,1,1]) # [x0, y0, width, height]
ax.bar(fruits,amount)
ax.legend(labels=['Amount'])
fig.show()

px.bar(x=fruits, y=amount)

Histogram

The histogram is used where the data is been distributed while the bar graph is used in comparing the two entities. Histograms are preferred during the arrays or data containing the long list. Consider an example where we can plot the age of the population with respect to the bin. The bin refers to the range of values divided into a series of intervals. In the below example bins are created with an interval of 10 which contains the elements from 0 to 9, then 10 to 19, and so on.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


population_age = [22,55,62,45,21,22,34,42,42,4,2,102,95,85,55,110,120,70,65,55,111,115,80,75,65,54,44,43,42,48]
bins = [0,10,20,30,40,50,60,70,80,90,100]

fig = plt.figure()
ax = fig.add_axes([0,0,1,1]) # [x0, y0, width, height]
ax.hist(population_age,bins=bins, rwidth=0.8)
ax.legend(labels=['Population'])
fig.show()

px.histogram(x=population_age, nbins=11)

Pie Chart

A pie chart is a circular graph that is divided into segments or slices of pie (Like a Pizza). It is used to represent the percentage or proportional data where each slice of the pie represents a category. This is very similar to Bar chart, but in a circular fashion.

When there is an order defined in the labels, or there are too many labels then we will use Bar charts. And when the label does not have any order, we can use Pie Charts.

We will use Pie Charts for attribute balance checking, categorical data proportion, etc. in a later lesson.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


fruits = ['Apple', 'Orange', 'Pineapple', 'JackFruit', 'Banana']
amount = [23,17,35,40,12]

fig = plt.figure()
ax = fig.add_axes([0,0,1,1])
ax.pie(amount, labels=fruits, startangle=90,shadow=True,explode=(0,0.2,0,0.1,0))
ax.legend(labels=['Fruits'])
fig.show()

px.pie(values=amount, names=fruits, title='Fruits Distribution')

Display Images

Both matplotlib and plotly can also display images, annotate them, show their histogram, and many more things. We will go through this when we introduce CNN (Convolutional Neural Network).

1
2
3
4
5
6
7
8


img = plt.imread('http://matplotlib.sourceforge.net/_static/logo2.png')

# Plotly
fig = px.imshow(img*255)
fig.show()

# matplotlib
plt.imshow(img)

Conclusion

There are numerous ways of visualization of your data. Later, we will also have a look at a clustering dataset, where we will extensively use scatter plots with colors. Till then, try to play around with different types of data. And finally, go through the tutorial of both matplotlib and plotly.

Machine Learning Tutorial - Lesson 07

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Machine Learning Made Easy: Scikit-Learn (Part-1: Basics and Supervised Learning)

Introduction

In this lesson, we will go through some common usages of the powerful and most popular Machine Learning framework, Scikit-Learn. The scikit-Learn library helps all the newcomers to learn more about different Machine Learning practices. It helps the user to achieve the most trivial machine learning tasks in the fewest lines possible. Although Scikit-Learn requires you to learn the very basics of Machine Learning (e.g. model fitting, predicting, cross-validation, etc.), we will learn those in the process of learning this awesome framework.

Installation

To install scikit-learn and scikit-image (a dependency for image processing with scikit-learn, we will use this later), just try the below command in your desired environment/package manager.

1
2
3
4
5
6
7


# pip
pip install -U scikit-learn
pip install -U scikit-image

# Conda
conda install -c conda-forge scikit-learn
conda install scikit-image

The Basics

Scikit-learn (sklearn) tries to divide everything related to Machine Learning into the following categories.

Supervised Learning:

a. Classification Problem

b. Regression Problem
Unsupervised Learning

a. Density Estimation

b. Clustering

c. Dimensionality Reduction

Based on these classifications, sklearn has tried a uniform approach for all the data, transformation, preprocessing, and models. In this lesson, we will only have a look at supervised learning. Let us discuss each of these one by one.

Handling Data with sklearn

Sklearn has a very mature dataset loading and preprocessing end-to-end working methods. It also has a few standard datasets, for instance, the iris and digits datasets for classification and the diabetes dataset for regression. Let us first load the first two datasets.

1
2
3
4


from sklearn import datasets

iris = datasets.load_iris()
digits = datasets.load_digits()

Sklearn datasets are of type Bunch, an internal dataset type for sklearn very similar to a dictionary. You can access the data directly by the attribute data which is a NumPy array. The shape of this array is always of (n_samples, n_features).

Of course, you can use any data from any source. We will discuss how and where to find data in another lesson. Assuming all the data will be a NumPy array, it will work with any function in sklearn.

Alongside learning about Scikit-learn, we will try to create a classification supervised learning algorithm with sklearn only. You will slowly see, how easy it is to create an end-to-end ML pipeline, with sklearn.

Loading Dataset

As we can see in the previous example, the digits data is of shape (1797, 64). This means that the data has n_sample=1797 samples, each having n_features=64 features. If you look at the dataset documentation, you will see that it has many attributes. Among them, the most interesting attributes are “data”, “target”, and “images”. Access them in your code and see what each of those are.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


# Visualize from image attribute
plt.gray()
plt.matshow(digits.images[0]) # Image of 0
plt.show()

# Visualize from data attribute
plt.matshow(digits.data[0].reshape(8, -1)) # Image of 0

# The target of the digit
digits.target[0] # 0

Preprocessing with Sklearn

For preprocessing the data, sklearn provides a separate submodule names sklearn.preprocessing. There are about 27 preprocessing functions included in scikit-learn (and still growing). First, we will have a look at the general structure and workflow of how a preprocessing step/function works in sklearn. The typical snippet for data processing goes like this:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15


# There is no class/function named processor, replace it with an actual processor of sklearn
from sklearn import preprocessing

# Get an instance of processor
processor_instance = preprocessing.Processor()

# Fit and Transform your data separately
processor_instance.fit(data)
transformed_data = processor_instance.transform(data).toarray()

# Or in one step
transformed_data = processor_instance.fit_transform(data).toarray()

# To inverse this process
data = processor_instance.inverse_transform(transformed_data)

There are many types of preprocessing, let us discuss some of the common ones we will use in this lesson. As there are tons of transformation functions, please go ahead and look at the API documentation to learn about all of them.

OneHot Encoding

As we were working with the digits dataset, let us transform the target labels into one-hot encoded labels. Here, one-hot encoding means we will assign a binary value for each of the categories of the label/feature.

1
2
3
4


onehot_encoder = preprocessing.OneHotEncoder()
encoded_labels = onehot_encoder.fit_transform(digits.target.reshape(-1, 1)).toarray()

encoded_labels.shape # (n_samples, 10)

Binarization of array

If you want an array to be binary, where all the nonzero values will be turned to 1, you can use the Binarizer preprocessor of sklearn. Without the help of sklearn, you would need to do something like this:

1
2


np_done = digits.data.copy()
np_done[np_done != 0] = 1

But, later you’ll know that this kind of processing is not robust, and you cannot use this is a complete ML pipeline made by sklearn. So, we will use sklearn.preprocessing.Binarizer for this.

1

binary_image = preprocessing.Binarizer().fit_transform(digits.data)

Binarizer will only replace the nonzero values to 1. If you want a more robust binarization like threshold, you can use the preprocessing function binarize directly without instantiating.

1

binary_image = preprocessing.binarize(digits.data, threshold=0.5)

Label Encoding

This preprocessing function is very important as it is used in many different algorithms to process data first. Although we do not need this preprocessing to build our digits classifier because the target is already label encoded, it is worth learning now!

Let’s imagine that our target of the digits object is not given like an array of [0..9]. They are given as a string of names of the digits. See the code below:

1
2
3
4


# Lets think we do not have digits.target at our hand
# Instead, we have "targets" like below

targets = ['one', 'zero', 'one', 'two', 'five', 'three', 'four', 'eight', 'six', 'nine', 'seven']

Then, how do we input this array of string into our model/classifier? We have to change the array, and replace all the values of “one"s into 1, “two"s into 2, and so on. This is what label encoded does.

1
2
3
4
5
6
7
8
9


le = preprocessing.LabelEncoder()

targets_encoded = le.fit_transform(targets)
print(targets_encoded) # [4 9 4 8 1 7 2 0 6 3 5]

# We use targets_encoded in our ML classification

targets_decoded = le.inverse_transform(targets_encoded)
print(targets_decoded) # Same as targets

Remember, the encoded label of values 0..9 are insignificant of the actual labels of the images we know. Here, an image of “1” can be encoded as 5 (because of the internal implementation of sklearn). But it won’t matter as long as all the images of “1” are encoded as 5. We can again inverse_transom to get the actual string again

Feature Scaling and Normalization

Previously, we have done feature scaling using two methods, min-max feature scaling and standardization. We did both of them using only NumPy. For this, we had to remember the equation and implement it every time. Sklearn has built-in preprocessor classes for both of these.

We will use this feature scaling for our digit classification. Note that, currently the range of the data is:

1

print('range is :', digits.data.min(), ',', digits.data.max()) # (0.0 , 16.0)

So we will apply min-max feature scaling on this to make them from 0 to 1. We are also showing the standardization, but as this data does not follow any gaussian distribution, we won’t be using it as a preprocessing step.

As you already know about these scaling methods, let’s just get familiarized with the classes only.

1
2
3
4
5
6
7


# Min-Max scaling
scaler = preprocessing.MinMaxScaler(feature_range=(0, 1), clip=True)
scaled_data = scaler.fit_transform(digits.data)

# Normalization
scaler = preprocessing.StandardScaler()
scaled_data = scaler.fit_transform(digits.data)

Custom Preprocessing Methods

Although sklearn offers tons of different preprocessing methods, we may need something else that is not available. Then, we can easily use preprocessing.FunctionTransformer to build our own preprocessor. This object takes a callable in its constructor which will do our required preprocessing. The callable must take the input X and return an output array whose first dimension must be equal to n_samples. We can also provide another function for the inverse transformation.

For example, there is no preprocessor to apply the log function on the data. We can easily build one ourselves.

1
2


log1p = preprocessing.FunctionTransformer(np.log1p)
logged_data = log1p.transform(digits.data)

Splitting Train and Test

In a previous lesson, we shuffled the dataset along with the target, and then took the first 80% of the data as train, 20% data as cross-validation. We can do all of these with only one line. Let’s split our digits dataset into train and cross-validation.

1
2
3
4


from sklearn.model_selection import train_test_split # Train Test split

# Shuffle and divide the data into two parts, the second one having 25% of the whole data (all with labels).
X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, test_size=0.25, random_state=42)

Classification with Sklearn

Now that we know how to load and preprocess the data, we can create several classifiers and train them on the data. Finally, we will also do model evaluation and select the best model with the best parameters.

A model in sklearn is another name for estimator. An estimator for classification is a Python object that implements the methods fit(X, y) and predict(T).

We will try some classifiers and see which one works best. First, let’s start with a Decision Tree classifier. We will use these classifiers as a black box as their mathematical background is quite complex and out of the scope of this series. But you are welcome to look into the details here.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13


from sklearn.tree import DecisionTreeClassifier # Our first classifier
from sklearn.metrics import accuracy_score # To calculate accuracy

# Build the classifier
clf = DecisionTreeClassifier(random_state=0)
# Now train it
clf.fit(X_train, y_train)

# Predict our cross-validation data
pred = clf.predict(X_test)

# Calculate accuracy of the model
print(accuracy_score(pred, y_test)) # 0.8688888888888889

Well, the accuracy is not very bad. Let us see if any other model does better or not.

Another example of an estimator is the class sklearn.svm.SVC, which implements support vector classification. The estimator’s constructor takes as arguments the model’s parameters. For now, we will consider the estimator as a black box:

1
2
3
4
5
6
7


from sklearn import svm
from sklearn.metrics import accuracy_score # To calculate accuracy

clf = svm.SVC(gamma=0.001, C=100.)
clf.fit(X_train, y_train)
pred = clf.predict(X_test)
print(accuracy_score(pred, y_test)) # 0.9911111111111112

Whoa, the accuracy is great! So it seems Support Vector Machine (SVM) classifiers worked better in this dataset. This is done without any preprocessing at all.

Selecting the right model with the right parameters

In the previous section, we used two models for predicting the digits. Decision Tree Classifier and Support Vector Machine. Again in SVM, we used two hyper-parameters, gamma and C. We can certainly tell that SVM is better for classifying this dataset. But there are some scenarios, where only the accuracy of the cross-validation set is not enough to compare two classifiers. Moreover, how are you sure that the value 0.001 for gamma and 100 for C is the best. We could change these hyper-parameters and get even better results.

All of these tasks are part of model evaluation. Every sklearn estimator exposes a score method that can quickly evaluate that classifier. Let’s see how to use it:

1
2
3


clf = svm.SVC(gamma=0.001, C=100.)
clf.fit(X_train, y_train)
clf.score(X_test, y_test)

So, for different values, we could do something like this:

1
2
3
4


for gamma, C in np.array(np.meshgrid([0.0001, 0.001, 0.005, 0.01, 0.1], [10, 50, 100, 200, 500])).T.reshape(-1,2):
 clf = svm.SVC(gamma=gamma, C=C)
 clf.fit(X_train, y_train)
 print(f"Gamma = {gamma}, C = {C} \t| Score: {clf.score(X_test, y_test)}")

Notice how complex it is to get all the combinations of different parameters for the classification score.

Grid Searching

The above implementation complexity can be avoided using the Grid-Search method of Scikit-Learn. We only need to provide the values we want to experiment with, and everything will be taken care of automatically.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19


from sklearn.model_selection import GridSearchCV, cross_val_score

gammas = [0.0001, 0.001, 0.005, 0.01, 0.1]
Cs = [10, 50, 100, 200, 500]

# We are wr
grid_searcher = GridSearchCV(
 estimator=clf,
 param_grid=dict(
 C=Cs,
 gamma=gammas
 ),
 n_jobs=-1
)

grid_searcher.fit(X_train, y_train)
print("Best Score is ", grid_searcher.best_score_)
print("Best C is ", grid_searcher.best_estimator_.C)
print("Best Gamma is ", grid_searcher.best_estimator_.gamma)

The output is:

1
2
3


Best Score is 0.988861352058378
Best C is 10
Best Gamma is 0.001

Without calculating combinations and running for all combinations in a for loop, we have achieved similar results much easier. With this method, we do not need to go through the result to search for the best score. It is already given to us.

Note that Grid Search of Scikit-Learn is not the same as we manually did in the for-loop. It is much more efficient in many ways. First of all, it does not run the estimator for all the parameter combinations. It applies a grid-search algorithm to slowly reach the optimal combination. For example, if the score of both (0.0005,10) and (0.005,10) for the value of (gamma, C) are worse than (0.001,10), the algorithm will never run it for any other value of gamma. Because 0.001 is found to be the optimal one. Secondly, it uses multiple cores of the CPU to run estimators in parallel which greatly speeds things up.

K-Fold Cross-Validation

Sometimes, a model/classifier works very well on a particular test data but does not work well on other test data. For example, our digit classifier may work well for this test data but may fail to predict correctly if we take the first 20% of data as a test. This is known as the residual problem.

To solve this uncertain accuracy problem, K-Fold Cross Validation is a great method. In K-Fold Cross Validation, the data set is divided into k subsets, and the holdout method is repeated k times. Each time, one of the k subsets is used as the test set and the other k-1 subsets are put together to form a training set. Then the average error across all k trials is computed. The advantage of this method is that it matters less how the data gets divided. Every data point gets to be in a test set exactly once and gets to be in a training set k-1 times. The variance of the resulting estimate is reduced as k is increased. The disadvantage of this method is that the training algorithm has to be rerun from scratch k times, which means it takes k times as much computation to make an evaluation.

Sklearn provides a lot of ways to perform cross-validation in your data. We are showing you only some popular methods here.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14


# Loop through K-folds and train yourself
accuracies = []
for train, test in KFold(n_splits=5).split(digits.data, digits.target):
 clf = svm.SVC(gamma=0.001, C=100.)
 clf.fit(digits.data[train], digits.target[train])
 accuracy = clf.score(digits.data[test], digits.target[test])
 accuracies.append(accuracy)
print(accuracies)

# One Liner of the above approach
[svm.SVC(gamma=0.001, C=100.).fit(digits.data[train], digits.target[train]).score(digits.data[test], digits.target[test]) for train, test in KFold(n_splits=5).split(digits.data, digits.target)]

# To become more automated without the loop run by ourselves
cross_val_score(svm.SVC(gamma=0.001, C=100.), digits.data, digits.target, cv=KFold(n_splits=5)) # Get mean to get final result

With these techniques, you can easily compare different estimators and get the best classifier for your dataset.

Solving different dataset problems with different approaches

As of the official guide of Scikit-learn, often the hardest part of solving a machine learning problem can be finding the right estimator for the job. Different estimators are better suited for different types of data and different problems.

The flowchart below is designed to give users a bit of a rough guide on how to approach problems concerning which estimators to try on your data.

End-to-End Pipelines

Suppose you are reading raw data from somewhere, preprocessing it, and then finally training it on a classifier. Maybe you are using that result to input into another classifier? Or you are doing multiple stages of preprocessing. After writing several functions, you want to apply your data to all of those functions. This time, again you will need to call all those functions separately. In the process, you need to maintain the flow order of the data and maintain the matching of shapes of data across different functions.

This sounds a bit cumbersome, right? Sklearn is here again to the rescue. Scikit-Learn has sklearn.pipeline to create one single call to apply all these functions sequentially, providing you what you only care about, the final predictions. In the code snippet below, we are creating a pipeline for our digit classifier and applying it to the test data.

1
2
3
4
5
6
7
8


from sklearn.pipeline import Pipeline

pipe = Pipeline(steps=[
 ('Feature Scaling', preprocessing.MinMaxScaler(feature_range=(0, 1), clip=True)),
 ('SVM', svm.SVC(gamma=0.001, C=10.))
])

cross_val_score(pipe, digits.data, digits.target, cv=KFold(n_splits=5))

Conclusion

In this lesson, we have become familiar with one of the most popular frameworks for machine learning. You will be glad to know that a lot of other ML-based frameworks like TensorFlow, PyTorch share very similar API (e.g. fit, predict). For this, learning scikit-learn after NumPy is the first most important step for any beginners. In the next lesson, we will start learning our final Framework in this series, TensorFlow.

Machine Learning Tutorial - Lesson 07.5

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Machine Learning Made Easy: Scikit-Learn (Part-2: Unsupervised Learning)

Introduction

Previously, we have understood the core concepts of supervised machine learning and how to use most of the common implementations of many preprocessing techniques, models, etc. with scikit-learn. In this lesson, we will be covering most approaches and techniques used for unsupervised learning using the power of scikit-learn.

The Basics

As a recap, Scikit-learn tries to divide everything related to Machine Learning into the following categories.

Supervised Learning:

a. Classification Problem

b. Regression Problem
Unsupervised Learning

a. Clustering

b. Dimensionality Reduction

c. Density Estimation

Based on these classifications, sklearn has tried a uniform approach for all the data, transformation, preprocessing, and models. In the previous lesson, we have gone through the supervised learning part in this category. In this lesson, we will have a look at unsupervised learning. Let us discuss each of these one by one.

What is unsupervised learning?

According to Wikipedia,

Unsupervised learning (UL) is a type of algorithm that learns patterns from untagged data.

So you will have data, where you do not have any target label to train on. For example, suppose you have 1000 images of cats and dogs, but the images are not labeled as which ones are cats and which ones are dogs. If you then try to create a partition and separate two animals using machine learning methods, that will be called unsupervised learning.

Most people think that unsupervised learning is a little harder than supervised learning. Because it is difficult to determine the loss or goodness of a model as there is no label to compare with. For clustering a dataset, you can use the distance between the mean of two clusters or inverse of distance among points in similar clusters as a goodness of the model. In this lesson, we will go through data clustering, outlier detection, data augmentation, synthesis, and many other unsupervised learning applications.

Clustering a dataset

Clustering is a very if not most popular algorithm definition for unsupervised learning. Clustering is the task of grouping a set of objects in such a way that objects in the same group are more similar to each other than to those in other groups. And each of these groups is called a cluster. For example, suppose you are at a party where everybody is wearing dresses of different colors. Now you are playing a game, where everyone with the same color dress, will come along together. So finally all people wearing red will be together, all persons wearing blue will stand together and same for the other colors. This is called clustering based on the color of each person’s dress.

In the example above, the whole game can a called a clustering algorithm. Here, the similarity between two persons (or objects) is the color. So color is the clustering feature. The difference between the color of two dresses is the distance between two objects while clustering. We could use other clustering features along with the color, for example where each person lives.

There are many clustering algorithms to use when clustering a dataset. We will start with the most common ones and show you how simple it is to implement one using scikit-learn.

Types of clustering

Previously, we have learned everything about K-means clustering. But there are hundreds of more clustering algorithms to use depending on your needs. From the top view, there are two types of clustering that you need to know.

Divisive Clustering: In this algorithm, we start from only one cluster. At each iteration, we divide clusters to make more clusters. For example, let’s say we have a dataset of 4 points ${A, B, C, D}$. We first divide that dataset into two clusters, ${{A,B}, {C, B}}$. Later, if needed
Agglomerative Clustering: In agglomerative clustering, we start by assuming each point as its own cluster. So if we have $n$ sample points, then we will start with $n$ clusters. At each iteration, we will merge the two closest clusters into one. Keep in mind that the definition of closest cluster may vary from application to application. But it must follow the Pythagorean distance formula. After some iterations, we will end up in only one cluster.

K-Means Clustering

Mathematically, given a set of observations $(x_1, x_2, …, x_n)$, where each observation is a d-dimensional real vector, k-means clustering aims to partition the $n$ observations into $k (≤ n)$ sets $S = {S_1, S_2, …, S_k}$ so as to minimize the within-cluster sum of squares (i.e. variance). Formally, the objective is to find:

$$arg\ min_s \sum_{i=1}^{k}{\sum_{x \in S_i}{||x - \mu_i||^2}} = arg\ min_s \sum_{i=1}^{k}{|S_i|}\ Var\ S_i$$

where $\mu_i$ is the mean of points in $S_i$. This is equivalent to minimizing the pairwise squared deviations of points in the same cluster:

$$arg\ min_s \sum_{i = 1}^{k}{\frac{1}{2 |S_i|}} \sum_{x,y \in S_i}{||x-y||^2}$$

The equivalence can be deduced from identity:

$$\sum_{x \in S_i}{||x-\mu_i||^2} = \sum_{x \neq y \in S_i}{(x-\mu_i)^T (\mu_i - y)}$$

Because the total variance is constant, this is equivalent to maximizing the sum of squared deviations between points in different clusters (between-cluster sum of squares, BCSS), which follows from the law of total variance.

If you are like me and do not understand anything written above, just see that last equation again and think of our objective to minimize the term with respect to $S$.

The unsupervised k-means algorithm has a loose relationship to the k-nearest neighbor classifier, a popular supervised machine learning technique for classification that is often confused with k-means due to the name.

Here, the term “k-means” means that we will have a total of $k$ points in the feature space. These k points are known as centroids of each cluster. Let’s first implement a K-means clustering from scratch. Our steps will be:

1
2
3
4
5


1. Randomly pick k data points as our initial Centroids.
2. Find the distance (Euclidean distance for our purpose) between each data points in our training set with the k centroids.
3. Now assign each data point to the closest centroid according to the distance found.
4. Update centroid location by taking the average of the points in each cluster group.
5. Repeat Steps 2 to 4 till our centroids don’t change.

K-Means from scratch

First, we will need a dataset. We will randomly generate a dataset for clustering. Luckily, sklearn gifts us a function named make_blob in its datasets module. This function generates isotropic Gaussian blobs for clustering experimentation.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


import numpy as np
from sklearn.datasets import make_blobs

n_samples = 1500
random_state = 170
X, y = make_blobs(n_samples=n_samples, random_state=random_state)

plt.subplot(221)
plt.scatter(X[:, 0], X[:, 1])
plt.title("Unclustered")

This will give us a plot of the unclustered dataset retrieved from make_blobs. Visually you can see there are 3 clusters in the dataset.

To cluster this dataset with K-means, we first need to randomly initialize 3 centroid points within the 2D space. We can do this by the following code:

1
2


#Randomly choosing Centroids
centroids = x[np.random.choice(len(x), k)]

After defining the centroids, we have to calculate the distance of all the points to these centroids. We could use a loop over all the points and calculate the distance manually like below:

1
2
3
4
5
6


distances = []
for point in X:
 dist_tmp = []
 for centroid in centroids:
 dist_tmp.append(sklearn.metrics.pairwise.euclidean_distances(point, centroids))
 distances.append(dist_tmp)

But luckily, sklearn provides a convenient function that does this whole thing in a one-line function call.

1
2
3


#finding the distance between centroids and all the data points
from scipy.spatial.distance import cdist
distances = cdist(x, centroids ,'euclidean')

Now that we have the distances of each point to all the centroids, we will mark each point as a member of that cluster, whose centroid is the closest. For example, a point has a distance like [5.5, 2.8, 8.3], so it is 5.5 away from centroid 0, 2.8 away from centroid 1, and 8.3 away from centroid 2. If we mark this based on the closest centroid, then the point should be marked as 1. So we can use the np.argmin function to get the index of the minimum distance and use it as centroid index.

1
2


#Centroid with the minimum Distance
points = np.array([np.argmin(i) for i in distances])

Now all the points are marked into a cluster. But this is just the first iteration (and initialization) of the K-means clustering algorithm. We have to do this for several iterations (say n_iterations).

1
2
3
4
5
6
7
8
9


for _ in range(n_iteration):
 centroids = []
 for idx in range(k):
 temp_cent = x[points==idx].mean(axis=0)
 centroids.append(temp_cent)

 centroids = np.vstack(centroids) # Updated Centroids
 distances = cdist(x, centroids ,'euclidean')
 points = np.array([np.argmin(i) for i in distances])

If we wrap the whole thing inside a function, it should look something like below. Also, this will work with any dimension of the points.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16


def kmeans(x, k, n_iteration):
 centroids = x[np.random.choice(len(x), k)]
 distances = cdist(x, centroids ,'euclidean')
 points = np.array([np.argmin(i) for i in distances])

 for _ in range(n_iteration):
 centroids = []
 for idx in range(k):
 temp_cent = x[points==idx].mean(axis=0)
 centroids.append(temp_cent)

 centroids = np.vstack(centroids) # Updated Centroids
 distances = cdist(x, centroids ,'euclidean')
 points = np.array([np.argmin(i) for i in distances])

 return points

After applying this function, we can see the result very impressive for our dummy blob dataset.

1
2
3
4
5


y_pred = kmeans(X, 3, 100)

plt.subplot(221)
plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.title("Our K-means implementation")

K-means with sklearn

All the things we did above can be done with sklearn far more easily. We instantiate a KMeans algorithm object/model and run it over our blob dataset.

1
2
3
4
5


y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X)

plt.subplot(221)
plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.title("K-Means sklearn")

The result is quite similar.

But for real data, we might even not know the potential number of clusters. There are many ways to also learn the number of clusters. We can choose the optimal value of K (Number of Clusters) using methods like the The Elbow method. Scikit-learn also has an implementation of this algorithm.

DBSCAN clustering

The full form of DBSCAN is “Density-Based Spatial Clustering of Applications with Noise”. So it is a density-based clustering non-parametric algorithm. Given a set of sample points, the algorithm groups together points that are closely packed together (points with many nearby neighbors), marking them as outliers points that lie alone in low-density regions (whose nearest neighbors are too far away). DBSCAN is one of the most common clustering algorithms and is also most cited in scientific literature.

The benefit of the DBSCAN algorithm is that it can identify the shape of a cluster and mark all points of that shape into a single cluster. The is not possible in the case of K-Means clustering algorithm. See the image below to understand the strength of DBSCAN.

As you can see, if the pattern of the sample points is a little complex, you should go for DBSCAN instead of K-Means clustering. Let us see how you can implement DBSCAN. You can see a more detailed example of this here.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28


from sklearn.cluster import DBSCAN
n_samples = 1500
X, y = make_blobs(n_samples=n_samples)
# Try to modify eps according to your needs
db = DBSCAN(eps=0.8, min_samples=2).fit(X)
core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
core_samples_mask[db.core_sample_indices_] = True
# Black removed and is used for noise instead.
unique_labels = set(db.labels_)
colors = [plt.cm.Spectral(each)
 for each in np.linspace(0, 1, len(unique_labels))]
for k, col in zip(unique_labels, colors):
 if k == -1:
 # Black used for noise.
 col = [0, 0, 0, 1]

 class_member_mask = (db.labels_ == k)

 xy = X[class_member_mask & core_samples_mask]
 plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
 markeredgecolor='k', markersize=14)

 xy = X[class_member_mask & ~core_samples_mask]
 plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
 markeredgecolor='k', markersize=6)

plt.title('DBSCAN clustering')
plt.show()

The output will look something like the below:

Other Clustering Algorithms

I am just showing you only 2 popular clustering algorithms. But there are a lot more. Each algorithm has its strength and weakness. A comparison and implementation of 10 popular clustering algorithms are shown in the below code. The clustering algorithms used for this code is the following:

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171


print(__doc__)

import time
import warnings

import numpy as np
import matplotlib.pyplot as plt

from sklearn import cluster, datasets, mixture
from sklearn.neighbors import kneighbors_graph
from sklearn.preprocessing import StandardScaler
from itertools import cycle, islice

np.random.seed(0)

# ============
# Generate datasets. We choose the size big enough to see the scalability
# of the algorithms, but not too big to avoid too long running times
# ============
n_samples = 1500
noisy_circles = datasets.make_circles(n_samples=n_samples, factor=.5,
 noise=.05)
noisy_moons = datasets.make_moons(n_samples=n_samples, noise=.05)
blobs = datasets.make_blobs(n_samples=n_samples, random_state=8)
no_structure = np.random.rand(n_samples, 2), None

# Anisotropicly distributed data
random_state = 170
X, y = datasets.make_blobs(n_samples=n_samples, random_state=random_state)
transformation = [[0.6, -0.6], [-0.4, 0.8]]
X_aniso = np.dot(X, transformation)
aniso = (X_aniso, y)

# blobs with varied variances
varied = datasets.make_blobs(n_samples=n_samples,
 cluster_std=[1.0, 2.5, 0.5],
 random_state=random_state)

# ============
# Set up cluster parameters
# ============
plt.figure(figsize=(9 * 2 + 3, 13))
plt.subplots_adjust(left=.02, right=.98, bottom=.001, top=.95, wspace=.05,
 hspace=.01)

plot_num = 1

default_base = {'quantile': .3,
 'eps': .3,
 'damping': .9,
 'preference': -200,
 'n_neighbors': 10,
 'n_clusters': 3,
 'min_samples': 20,
 'xi': 0.05,
 'min_cluster_size': 0.1}

datasets = [
 (noisy_circles, {'damping': .77, 'preference': -240,
 'quantile': .2, 'n_clusters': 2,
 'min_samples': 20, 'xi': 0.25}),
 (noisy_moons, {'damping': .75, 'preference': -220, 'n_clusters': 2}),
 (varied, {'eps': .18, 'n_neighbors': 2,
 'min_samples': 5, 'xi': 0.035, 'min_cluster_size': .2}),
 (aniso, {'eps': .15, 'n_neighbors': 2,
 'min_samples': 20, 'xi': 0.1, 'min_cluster_size': .2}),
 (blobs, {}),
 (no_structure, {})]

for i_dataset, (dataset, algo_params) in enumerate(datasets):
 # update parameters with dataset-specific values
 params = default_base.copy()
 params.update(algo_params)

 X, y = dataset

 # normalize dataset for easier parameter selection
 X = StandardScaler().fit_transform(X)

 # estimate bandwidth for mean shift
 bandwidth = cluster.estimate_bandwidth(X, quantile=params['quantile'])

 # connectivity matrix for structured Ward
 connectivity = kneighbors_graph(
 X, n_neighbors=params['n_neighbors'], include_self=False)
 # make connectivity symmetric
 connectivity = 0.5 * (connectivity + connectivity.T)

 # ============
 # Create cluster objects
 # ============
 ms = cluster.MeanShift(bandwidth=bandwidth, bin_seeding=True)
 two_means = cluster.MiniBatchKMeans(n_clusters=params['n_clusters'])
 ward = cluster.AgglomerativeClustering(
 n_clusters=params['n_clusters'], linkage='ward',
 connectivity=connectivity)
 spectral = cluster.SpectralClustering(
 n_clusters=params['n_clusters'], eigen_solver='arpack',
 affinity="nearest_neighbors")
 dbscan = cluster.DBSCAN(eps=params['eps'])
 optics = cluster.OPTICS(min_samples=params['min_samples'],
 xi=params['xi'],
 min_cluster_size=params['min_cluster_size'])
 affinity_propagation = cluster.AffinityPropagation(
 damping=params['damping'], preference=params['preference'])
 average_linkage = cluster.AgglomerativeClustering(
 linkage="average", affinity="cityblock",
 n_clusters=params['n_clusters'], connectivity=connectivity)
 birch = cluster.Birch(n_clusters=params['n_clusters'])
 gmm = mixture.GaussianMixture(
 n_components=params['n_clusters'], covariance_type='full')

 clustering_algorithms = (
 ('MiniBatch\nKMeans', two_means),
 ('Affinity\nPropagation', affinity_propagation),
 ('MeanShift', ms),
 ('Spectral\nClustering', spectral),
 ('Ward', ward),
 ('Agglomerative\nClustering', average_linkage),
 ('DBSCAN', dbscan),
 ('OPTICS', optics),
 ('BIRCH', birch),
 ('Gaussian\nMixture', gmm)
 )

 for name, algorithm in clustering_algorithms:
 t0 = time.time()

 # catch warnings related to kneighbors_graph
 with warnings.catch_warnings():
 warnings.filterwarnings(
 "ignore",
 message="the number of connected components of the " +
 "connectivity matrix is [0-9]{1,2}" +
 " > 1. Completing it to avoid stopping the tree early.",
 category=UserWarning)
 warnings.filterwarnings(
 "ignore",
 message="Graph is not fully connected, spectral embedding" +
 " may not work as expected.",
 category=UserWarning)
 algorithm.fit(X)

 t1 = time.time()
 if hasattr(algorithm, 'labels_'):
 y_pred = algorithm.labels_.astype(int)
 else:
 y_pred = algorithm.predict(X)

 plt.subplot(len(datasets), len(clustering_algorithms), plot_num)
 if i_dataset == 0:
 plt.title(name, size=18)

 colors = np.array(list(islice(cycle(['#377eb8', '#ff7f00', '#4daf4a',
 '#f781bf', '#a65628', '#984ea3',
 '#999999', '#e41a1c', '#dede00']),
 int(max(y_pred) + 1))))
 # add black color for outliers (if any)
 colors = np.append(colors, ["#000000"])
 plt.scatter(X[:, 0], X[:, 1], s=10, color=colors[y_pred])

 plt.xlim(-2.5, 2.5)
 plt.ylim(-2.5, 2.5)
 plt.xticks(())
 plt.yticks(())
 plt.text(.99, .01, ('%.2fs' % (t1 - t0)).lstrip('0'),
 transform=plt.gca().transAxes, size=15,
 horizontalalignment='right')
 plot_num += 1

plt.show()

The resulting image is like below:

Dimensionality Reduction

Before understanding the importance of data visualization, let us see 2 scenarios first:

Suppose you have a dataset with more than 3 attributes and you want to visualize them. How can you do that in a 2D or 3D space? The most obvious answer is to select only 2/3 attributes and visualize using them. You can also create multiple plots each representing 2/3 attributes of the dataset.
Suppose you have a very dataset with millions of attributes and you want to train a large model (say a neural network) using that data. You find that the training and prediction of the model are very slow and won’t fit your needs. So you can remove some attributes from the dataset and then use them for the model. Now, which attributes will you remove first? What if some of the attributes are more important than others? How do you determine that?

In these scenarios, a dimension reduction will help you a lot. You can create a new dataset from these where the number of attributes is less. You can also get the importance of each attribute in your dataset. Here the importance is derived from how much the dataset is diverse based on that attribute.

There are a lot of algorithms for Dimensionality Reduction. We will discuss only two of these methods: PCA and t-SNE.

Principal Components Analysis

Principal Component Analysis, or PCA, is a dimensionality-reduction method that is often used to reduce the dimensionality of large data sets, by transforming a large set of variables into a smaller one that still contains most of the information in the large set. The main objective of PCA is to reduce the number of variables of a data set while preserving as much information as possible. The steps of PCA is as follows:

Standardize the Dataset
Compute the Covariance Matrix
Compute Eigen Values and Eigen Vectors from Covariance Matrix
Sort the attributes according to their respecting Eigen Values and select the first required number of columns.

After standardizing the dataset, all the variables will be transformed to the same scale. So the range of an attribute will not apply any effect on PCA. The covariance matrix is a $nxn$ symmetric matrix (where $n$ is the number of dimensions) that has as entries the covariances associated with all possible pairs of the initial variables. For example, for a 3-dimensional data set with 3 variables x, y, and z, the covariance matrix is a $3x3$ matrix of this from:

$$\begin{bmatrix}
Cov(x,x) & Cov(x,y) & Cov(x,z)\
Cov(y,x) & Cov(y,y) & Cov(y,z)\
Cov(z,x) & Cov(z,y) & Cov(z,z)
\end{bmatrix}$$

After getting the covariance matrix, we calculate the eigenvalues and eigenvectors. We are going to have $n$ eigenvalues $\lambda_1, \lambda_2 … \lambda_n$. They are obtained by solving the equation given in the expression below:

$$|A - \lambda I_n| = 0$$

Here $I_n$ is an identity matrix of shape $nxn$. The corresponding eigenvectors $e_1, e_2… e_n$ are obtained by solving the expression below:

$$(A - \lambda_j I_n) e_j = 0$$

Here, we have the difference between the matrix $A$ minus the $j^{th}$ eigenvalue times the Identity matrix, this quantity is then multiplied by the $j^{th}$ eigenvector and set it all equal to zero. This will obtain the eigenvector $e_j$ associated with eigenvalue $\mu_j$.

PCA from scratch

Luckily, NumPy has all the functions to calculate these in one line. Let us implement this in code.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13


# mean Centering the data 
X_meaned = X - np.mean(X , axis = 0)

# calculating the covariance matrix of the mean-centered data.
cov_mat = np.cov(X_meaned , rowvar = False)

#Calculating Eigenvalues and Eigenvectors of the covariance matrix
eigen_values , eigen_vectors = np.linalg.eigh(cov_mat)

#sort the eigenvalues in descending order
sorted_index = np.argsort(eigen_values)[::-1]
sorted_eigenvalue = eigen_values[sorted_index]
sorted_eigenvectors = eigen_vectors[:,sorted_index]

After that, we can create a subset of our attributes and eigenvectors. We can also transform our original data using the eigenvectors.

1
2
3
4
5
6
7


# select the first n eigenvectors, n is desired dimension
# of our final reduced data.
eigenvector_subset = sorted_eigenvectors[:,0:n_components]

# We can also create the reduced dataset that has lower dimension and represents the actual
# dataset as much as possible
X_reduced = np.dot(eigenvector_subset.transpose() , X_meaned.transpose() ).transpose()

If we wrap the whole process into a function, the whole code should look something like this. We are applying the algorithm on our well-known Iris dataset taken from sklearn.dataset submodule. Iris has 4 dimensions, we will reduce it to 2 dimensions for visualization in a 2D plot.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39


def PCA(X , n_components):

 # mean Centering the data 
 X_meaned = X - np.mean(X , axis = 0)

 # calculating the covariance matrix of the mean-centered data.
 cov_mat = np.cov(X_meaned , rowvar = False)

 #Calculating Eigenvalues and Eigenvectors of the covariance matrix
 eigen_values , eigen_vectors = np.linalg.eigh(cov_mat)

 #sort the eigenvalues in descending order
 sorted_index = np.argsort(eigen_values)[::-1]
 sorted_eigenvalue = eigen_values[sorted_index]
 sorted_eigenvectors = eigen_vectors[:,sorted_index]

 # select the first n eigenvectors, n is desired dimension
 # of our final reduced data.
 eigenvector_subset = sorted_eigenvectors[:,0:n_components]

 # We can also create the reduced dataset that has lower dimension and represents the actual
 # dataset as much as possible
 X_reduced = np.dot(eigenvector_subset.transpose() , X_meaned.transpose() ).transpose()

 return X_reduced



iris = datasets.load_iris()
X = iris.data
y = iris.target
n_components = 2
X_reduced = PCA(X, n_components)

# For visualization
principal_df = pd.DataFrame(X_reduced , columns = ['PC1','PC2'])

# Using seaborn for cute display
sb.scatterplot(data = principal_df , x = 'PC1',y = 'PC2' , hue = y , s = 60)

The result is quite impressive as we can see three types of plants being partitions into their own spaces. This is the power of PCA.

PCA using sklearn

Now lets see the same in scikit-learn.

1
2


from sklearn.decomposition import PCA
X_reduced = PCA(n_components=2).fit(X).transform(X)

DONE. It is just that much simple.

t-SNE

When it comes to Dimensionality Reduction, PCA is famous because it is simple, fast, easy to use, and retains the overall variance of the dataset. Though PCA is great, it does have some drawbacks. One of the major drawbacks of PCA is that it does not retain non-linear variance. This means PCA will not be able to get good results for the dataset when you want to preserve the overall structure of the dataset.

In PCA, the relative distance between two sample points is not maintained, thus the dataset structure is deformed. But using t-SNE, you can reduce the dimension of the dataset preserving the relative distance between each sample point.

t-SNE is a nonlinear dimensionality reduction technique that is well suited for embedding high dimension data into lower dimensional data (2D or 3D) for data visualization using t-Distribution. t-SNE stands for t-distributed Stochastic Neighbor Embedding. Let us break each letter and try to understand them.

Stochastic: Not definite but random probability
Neighbor: Concerned only about retaining the variance of neighbor points
Embedding: Plotting data into lower dimensions

To apply t-SNE, we need to follow these steps:

Construct a probability distribution on pairs in higher dimensions such that similar objects are assigned a higher probability and dissimilar objects are assigned a lower probability.
Replicate the same probability distribution on lower dimensions iteratively till the Kullback-Leibler divergence is minimized.

We will not implement any more algorithms from scratch and make this lesson unnecessarily long. So let use t-SNE using scikit-learn.

1
2
3


from sklearn.manifold import TSNE
X_reduced = TSNE(n_components=2).fit_transform(X)
# Visualize

The visualization part is the same as above. The output of the algorithm on the Iris dataset is given below:

As you can see, the two labels are very close to each other and hard to distinguish. The later label can be very easily separated from the dataset. You do not get this distance between each sample using the PCA algorithm.

One important thing about t-SNE is its perplexity. This single hyperparameter can change the whole transformation of the dataset. The larger the perplexity, the more non-local information will be retained in the dimensionality reduction result.

Let us see the effect of different perplexity values on the IRIS dataset.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38


import numpy as np
import matplotlib.pyplot as plt

from matplotlib.ticker import NullFormatter
from sklearn import manifold, datasets
from time import time

n_samples = X.shape[0]
n_components = 2
(fig, subplots) = plt.subplots(1, 5, figsize=(20, 4))
perplexities = [1, 5, 30, 50, 100]

iris = datasets.load_iris()
X = iris.data
y = iris.target

red = y == 0
green = y == 1
blue = y == 2

for i, perplexity in enumerate(perplexities):
 ax = subplots[i]

 t0 = time()
 tsne = manifold.TSNE(n_components=n_components, init='random',
 random_state=0, perplexity=perplexity)
 Y = tsne.fit_transform(X)
 t1 = time()
 print("circles, perplexity=%d in %.2g sec" % (perplexity, t1 - t0))
 ax.set_title("Perplexity=%d" % perplexity)
 ax.scatter(Y[red, 0], Y[red, 1], c="r")
 ax.scatter(Y[green, 0], Y[green, 1], c="g")
 ax.scatter(Y[blue, 0], Y[blue, 1], c="b")
 ax.xaxis.set_major_formatter(NullFormatter())
 ax.yaxis.set_major_formatter(NullFormatter())
 ax.axis('tight')

plt.show()

The result is given below:

As you can see, the perfect perplexity stays between 30 to 50. Less perplexity pays more attention to the global information and increases the distance of some points from all other points too high. On the other hand, large perplexity pays a lot of attention to local information and tries to keep the distance of points in the same cluster higher than the intercluster distance.

There is also a very popular dimension reduction method. It is based on deep neural networks and is known as AutoEncoders. We will learn this in a later lesson after we cover the basics of Neural Networks.

Some words about Reinforcement Learning

Reinforcement Learning is another type of Machine Learning technique that is neither Supervised Learning nor Unsupervised Learning. Reinforcement learning is the training of machine learning models to make a sequence of decisions. Via this algorithm computers learn to achieve a goal in an uncertain, potentially complex environment. In reinforcement learning, artificial intelligence faces a game-like situation.

Suppose you are in a game environment. At first, you do not even know your goal. You do not what to do, and how to complete the game. Slowly you move forward and see a coin standing in a corner. You proceed to the coin and take it. The moment you pick up the coin, you see your score going up. You know that a higher score is always better. So you decide to take as much coin as possible throughout the game.

Again you see a thorn mounted on a wall. You try to touch that thorn and suddenly see your health go down. Eventually, if you keep touching that, your health will deplete and you will die. After understanding this, you decide to stay away from those for the entire game.

A virtual agent in the algorithm is just like you. He gets a reward in the game when moved towards a goal, and gets punished when moves away from the goal. There is a lot of live competition happening for reinforcement learning. For example, there are Halite.io, crowdanalytix, codalab, ods.ai etc.

Solving different dataset problems with different approaches

The flowchart below is designed to give users a bit of a rough guide on how to approach problems concerning which estimators to try on your data.

Conclusion

This lesson is the wrapping up of the Scikit-learn framework in Machine Learning. Now you can be confident that you can get started with both supervised and unsupervised learning algorithms. In a later lesson, when we will learn about deep learning algorithms, you will understand that the basic ideas of these two lessons on scikit-learn still apply. Apply some algorithms to your own needs and share that awesomeness in the discussion section.

Machine Learning Tutorial - Lesson 08

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Neural Networks

Introduction

In this lesson, we will introduce the hot cake of the field of Machine Learning. Neural Networks are models designed based on our brains. Human brains have about 86 billion cells connected together by synapses. A neural network also has many cells/units that are connected to one another. Whenever an input is given, it is propagated through the entire network of cells, reaching the output cells where certain actions are handled for certain outputs.

This lesson will help you build a neural network from scratch. Later, we will introduce TensorFlow which is the de facto standard for neural networks. So let’s get started.

How do human brains work?

Nerve cells are at the center of everything when it comes to brain functionality. A nerve cell neuron is a special biological cell that processes information. According to an estimation, there is a huge number of neurons, approximately $10^{11}$ with numerous interconnections. There are basically 4 parts of a neuron, then helps it to process and transfer information:

Synapses: Synapse is the connection point of two nerve cells.
Dendrites: They are tree-like branches, responsible for receiving the information from other neurons it is connected to. In another sense, we can say that they are like the ears of neurons.
Soma: It is the cell body of the neuron and is responsible for the processing of information, they have received from dendrites.
Axon: With this part, a cell transfers information to another cell (like wires).

Here is an image showing the connection of two neurons and their different parts.

This is not a biology lesson, right? So why are we learning about brains? Because neural networks are nothing but brains created by humans. Below is the analogy in neural networks of what we have read about brains earlier:

Synapses are weights in an artificial neural network (ANN).
Dendrites are the input of a cell in ANN.
Soma is a node or unit cell in ANN.
Axon is the output of a cell in ANN.

Artificial Neuron

A neuron is a unit for computation in ANN. An ANN is nothing but a large network of neurons. There are different types of neurons, and there are many types of connections between neurons. All these varieties make different kinds of deep neural networks, some work well for tabular data, others work for images, and some work for sequential data like texts or speech.

Here is the anatomy of a neuron in ANN.

Let us suppose the input has n features. So the input is like (X1, X2, X3… Xn). When this input data (more precisely a tensor) is given to the neural network, each of the features is multiplied by a weight. Then, all of them are summed up along with an additional bias value. This sum is the output of the neuron. Most of the time, to control this sum, an activation function is used. We will go through all of these in more detail.

$$a_1 = X_1 * W_1$$

$$a_2 = X_2 * W_2$$

$$a_3 = X_3 * W_3$$

$$…$$

$$a_n = X_n * W_n$$

The above can be written as:

$$a_i = X_i * W_i\ ;\ i \in {1, 2.. n}$$

Linear Layer

You might be thinking that we have to loop through all the features and then multiply them with the corresponding weights to get the output. But this will take forever for a very simple task. Remember, there could be thousands of features in an input (think of an image of size 4000*2000, then the number of pixels is 8000000). And this is only for one neuron. A modern ANN contains hundreds of such neurons.

The power of parallel processing with TPUs and GPUs has made this computation very fast and efficient. Graphics Processing Units (GPU) and Tensor Processing Units (TPU) can do matrix manipulation almost as fast as the CPU does simple arithmetics between two numbers. So we need to convert the above equations into matrix algebra.

Think of the input as a matrix where each column represents a feature and each row represents a sample.

Now we need to multiply all the features by the same weight for all samples. So we need to multiply each column by a value. If we consider a vector of weights of the same size as the number of features in input, then we can do matrix multiplication among them to get the desired result along with the summation.

But what about the BIAS we talked about??

The bias can also be added very easily in this matrix multiplication. That bias is appended to the weight vector in the first place. And a column (like a new feature) of 1 is appended in the first of the matrix $X$.

We have successfully understood the most basic part of deep learning, a layer of many unit cells. This is known as a linear layer. In deep learning, there will be a lot of layers joined together. Let’s see how we can implement it in python.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24


class LinearLayer():
 # We will initialize W and b. They should be very small, so multiplying by 0.1
 def __init__(self, input_size, output_size, activation=None, name=None):
 self.W = np.random.randn(output_size, input_size) * 0.1
 self.b = np.random.randn(output_size, 1) * 0.1

 # Activation function are in the next section
 self.activation = activation
 if not activation:
 self.activation = IdentityActivation()

 # The is the prediction of the layer.
 def forward(X):
 self.A_prev = X.copy()
 self.Z_curr = np.dot(self.W, X) + self.b
 self.A_curr = self.activation.forward(self.Z_curr)
 return self.A_curr.copy()

## For our test purpose
if __name__ = '__main__':
 ll1 = LinearLayer(4, 1)
 print(ll1.forward([
 [1, 2, 3, 4]
 ])

Activation Function

As we said earlier, we also need activation functions to keep the output data in a diserable shape. There are a lot of activation functions used in deep learning. Let us discuss a few popular of them.

Identity Function

This is the simplest activation function. Sometimes you do not need to use any activation function for a specific layer. But to hold consistency across all layers, you need to create and store a dummy activation function for that layer. That activation function is the Identity function, which outputs whatever it is given as input.

$$f(x) = x$$

1
2
3


class IdentityActivation():
 def forward(X):
 return X

Linear

When the output value is proportional to the input value anywhere in the input domain, then the function is said to be a linear function. So, a linear function is just multiplying something with the input.

$$f(x) = a*x$$

1
2
3
4
5
6


class LinearActivation():
 def __init__(self, a):
 self.a = a

 def forward(X):
 return a*X

Sigmoid

Now the activation function will start to get interesting. The main objective of a sigmoid is to keep the output data between 0 and 1. The equation of this function is:

$$f(x) = \frac{1}{1+e^{-x}}$$

1
2
3


class SigmoidActivation():
 def forward(X):
 return 1/(1+np.e**(-X))

Tanh

This activation function is very similar to sigmoid. The only difference is that the range of this function is [-1, 1]. And the equation used tanh function.

$$f(x) = tanh(x)$$

1
2
3


class TanhActivation():
 def forward(X):
 return np.tanh(X)

Binary Step

To make sure that the output is binary, a binary step activation function is used. This is mostly used when you are doing a binary classification. When the input is negative, the activation function outputs 0. And when it is positive, it outputs 1.

$$f(x) = 0\ if\ x \le 0\ ; 1\ otherwise$$

1
2
3
4
5
6
7
8


class BinaryStepActivation():
 def __init__(self, thresh=0):
 self.thresh = thresh

 def forward(X):
 out = np.ones_like(X)
 out[X <= 0] = 0
 return out

ReLU

Rectified Linear Unit Activation function has recently become popular for working very well with image datasets. It filters all the negative values to 0. For acts as a Linear Activation function for all the positive values.

$$f(x) = 0\ if\ x \le 0\ ;\ f_{linear}(x)\ otherwise$$

1
2
3
4
5
6
7


def ReLUActivation():
 def __init__(self, a):
 self.a = a
 def forward(X):
 out = X.copy()
 out[X <= 0] = 0
 out *= self.a

Leaky ReLU

Leaky ReLU is similar to ReLU, but it acts like another activation function instead of 0 when X is negative.

$$f(x) = bx\ if\ x \le 0\ ;\ ax(x)\ otherwise$$
$$b < a$$

1
2
3
4
5
6
7
8


class LeakyReLUActivation():
 def __init__(self,a,b):
 self.a = a
 self.b = b
 def forward(X):
 out = X.copy()
 out[X <= 0] *= self.b
 out[X > 0] *= self.a

SoftMax

SoftMax is a very different kind of activation function mostly used for multiclass classification. It takes all the outputs and then converts them to probability. The output of the softmax function will always add up to 1. And all the values of the output will be always non-negative.

$$f(x_i) = \frac{e^{x_i}}{\sum_{j=1}^J{e^{x_j}}}$$

As this is not a continuous function and depends on the individual value of x, it is quite difficult to plot on a 2D surface.

1
2
3


class SoftMaxActivation():
 def forward(X):
 return np.e**x / np.sum(np.e**x)

Neural Network with Feed Forward

So we have created linear layers and activation functions with forward methods. Our final class will be to create a NeuralNet which will contain everything it needs and provide an API to the user.

We will follow the Scikit-Learner API style for this. So there will be a predict method and a fit method (let’s not worry about all other utility methods for now). As we have just implemented the forward methods for all the classes, let’s create the predict method using those forward methods of all classes.

1
2
3
4
5
6
7
8


class NeuralNet():
 def __init__(self, layers=[]):
 self.layers = layers

 def predict(X):
 for layer in layers:
 A = layer.forward(A)
 return A

This looks easy, right? But now we will implement a little difficult part of the NeuralNet. The Backward Propagation.

Loss function

While training our neural network, we will need to calculate the loss depending on the prediction of the network, and target labels. We can simply use a function to implement the loss function equation of categorical cross-entropy.

1
2
3
4


def categorical_crossentropy(target, output):
 output /= output.sum(axis=-1, keepdims=True)
 output = np.clip(output, 1e-7, 1 - 1e-7) # So that weights dont become 0
 return np.sum(target * -np.log(output), axis=-1, keepdims=False)

Backward Propagation

We have already learned about the gradient descent algorithm in lesson 4. We will discuss the same and apply it to the neural networks backpropagation method. We already have the NeuralNet class along with other layers. All we need to do is add a new method called backward to each of them so that it is possible to do backward propagation to the network.

What will the backward method do in each case? The backward method will output the gradient of the input, with respect to W. The input we just mentioned is not X, it is the gradient we got from another layer.

Remember, gradient descendent algorithm and backward propagation are different algorithms. The first one is used only to get the gradient of the loss with respect to W. The second one is responsible for updating the weights using that gradient taken from the first algorithm.

You will understand this better if you take a look at the following equations. Nerd Alert! Calculus Ahead.

$$dW^{[l]} = \frac{\partial L}{\partial W^{[l]}} = \frac{1}{m} dZ^{[l]} A^{[l-1] T}$$

$$db^{[l]} = \frac{\partial L}{\partial b^{[l]}} = \frac{1}{m} dZ^{l} $$

$$dA^{[l-1]} = \frac{\partial L}{\partial A^{[l-1]}} = W^{[l] T} dZ^{[l]}$$

$$dZ^{[l]} = dA^{[l]} * g’(Z[l])$$

After calculating gradients of all the layers and activation functions using the above equations, we can update the weights via:

$$W^{[l]} = W^{[l]} - \alpha dW^{[l]}$$

$$b^{[l]} = b^{[l]} - \alpha db^{[l]}$$

Let us implement this in python. We are only implementing it on the LinearLayer class, sigmoid, and relu activation functions. You can look at the derivative of other activation functions here.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28


class SigmoidActivation():
 # ...
 def backward(self, A):
 # Just the derivative
 return A*(1-A)

class ReLUActivation():
 # ...
 def backward(self, A):
 A = A.copy()
 A[A <= 0] = 0;
 A *= self.a
 return A;

class LinearLayer():
 # ...
 def backward(dA_curr, lr=0.001):
 m = A_prev.shape[1]

 dZ_curr = self.activation.backward(dA_curr, self.Z_curr)
 dW_curr = np.dot(dZ_curr, self.A_prev.T) / m
 db_curr = np.sum(dZ_curr, axis=1, keepdims=True) / m
 dA_prev = np.dot(self.W.T, dZ_curr)

 self.W -= lr * dW_curr
 self.b -= lr * db_curr

 return dA_prev

Now that we have the method backward for all classes, we can implement the fit method in our NeuralNet class.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


class NeuralNet():
 # ...
 def fit(X, y, lr=0.001, epoch=1):
 for i in range(epoch):
 for layer in self.layers:
 X = layer.forward(X)

 dA_curr = categorical_crossentropy(y, X)

 for layer in reversed(self.layers):
 dA_curr, = layer.backward(dA_curr, lr=self.lr)

Now the only work left is to instantiate and train the model. We will train it on the iris dataset we discussed in the Scikit-learn chapter.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17


from sklearn.datasets import load_iris
from sklearn.metrics import accuracy_score # To calculate accuracy

iris = load_iris()

net = NeuralNet([
 LinearLayer(4, 10, activation=ReLUActivation()),
 LinearLayer(10, 5, activation=ReLUActivation()),
 LinearLayer(5, 3, activation=SoftMaxActivation()),
])

# We do preprocessing like the last lesson
X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.25, random_state=42)

net.fit(X_train, y_train)
pred = net.predict(X_test)
print(accuracy_score(pred, y_test))

Conclusion

Congratulations. You can successfully create a full-fledged Neural Network architecture. The network has many activation functions, can input 2-dimensional data with any shape, and can be as deep or as wide as you need.

I understand that all of the code and the mathematical backgrounds are tough for you. In real life, you do not have to do any of these at all. In the next lesson, we will learn TensorFlow. Then you will realize how easy it is to create and train a deep learning model within as small as 10 lines of code.

Machine Learning Tutorial - Lesson 09

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Introduction to TensorFlow

Introduction

Those who are continuing from the last lesson (Deep Learning) of this series, you might felt deep learning to be a very difficult concept. I agree with you, deep learning is not that easy if you try to implement it from scratch (reinvent the wheel). But in this lesson, we will walk through the final framework in this series, TensorFlow.

Installation

Installing TensorFlow 2 is easy. I will recommend everyone to install TF from pip even if you are using anaconda. Just activate the environment and install TensorFlow with pip. Deep learning algorithms require TPU or GPU to work at a feasible speed. And Tensorflow now natively supports only NVidia graphics cards. NVidia provides API named CUDA and CUDnn for tensor computation which is used by TensorFlow. But these two libraries are required to be installed separately on the machine. pip just installs TensorFlow. But conda can install both TensorFlow with optional CUDA and CUDnn library. In many operating systems (e.g. Linux and Mac), CUDA and CUDnn for conda versions do not work out of the box. That is why I am recommending you to install just TensorFlow with pip, and install CUDA, CUDnn for GPU support.

1
2
3
4
5
6


# pip
pip install tensorflow

# Conda
conda activate <your_env>
pip install tensorflow

But the main difficulty comes when you try to enable GPU support for TensorFlow.

Enable NVidia GPU support for TensorFlow 2

To make TensorFlow compatible with GPU, you need to install CUDA and CUDnn on your machine. Not all versions of CUDA are supported by all versions of TensorFlow. Below is a table for CUDA and TensorFlow 2 compatibility.

Version	Python version	cuDNN	CUDA
tensorflow-2.5	3.6-3.9	8.1	11.2
tensorflow-2.4	3.6-3.8	8.0	11.0
tensorflow-2.3	3.5-3.8	7.6	10.1
tensorflow-2.2	3.5-3.8	7.6	10.1
tensorflow-2.1	2.7, 3.5-3.7	7.6	10.1
tensorflow-2.0	2.7, 3.3-3.7	7.4	10.0

Note: If you are familiar with docker, you can just run docker run -it tensorflow/tensorflow bash to get started with TensorFlow with GPU support on any machine.

At the time of writing, the latest version of TensorFlow is 2.5.0. So we will install that version with CUDA 11.2 and cuDNN 8.1.

Here are all the things you need to download first. You may need to create an NVidia developer account to download cuDNN.

Install the driver and CUDA on your machine. Then extract cuDNN and paste the 3 folders into the directory C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\. If you installed CUDA on another directory, change the path accordingly.

You also need to add a new environment variable named “CUDA_PATH” which will point to “C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2”.

For complete guidance on installing cuDNN in different Operating Systems, you may follow the official installation guide

Finally, if you have trouble installing on your machine, or you have a less supported operating system (e.g. Arch Linux, Fedora, etc.), you can follow this guide to install from source.

After installation, test your installation of TensorFlow 2 and GPU support with the following code:

1
2
3
4
5


import tensorflow as tf
if tf.test.is_gpu_available() and tf.test.is_gpu_available(cuda_only=True) and tf.test.is_gpu_available(True, (3,0)):
 print("My GPU supports TensorFlow")
else:
 print("My GPU may be not compatible with TF2")

Tensor Processing in TensorFlow

Tensorflow is built to compute different arithmetics of tensors. In this section, we will start just like we started with NumPy many lessons ago. TensorFlow tries to be as much compatible with NumPy as possible. Although many NumPy features are missing in TensorFlow (like array indexing with array), you will automatically feel at home in TensorFlow if you know NumPy well.

Let’s create a tensor first.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30


import tensorflow as tf

a = tf.constant([1, 2, 3, 4, 5])
b = tf.convert_to_tensor([6, 7, 8, 9, 10]) # same as above

# All the operations like numpy
print(a + b)
print(a - b)
print(a * b)
print(a / b)
print(a ** b)
print(a & b)
print(a | b)
print(-a)
tf.matmul(a, b)
tf.transpose(a)
tf.exp(a)
tf.log(a)

# It has also many operation methods like NumPy
tf.abs(a) # Absolute
tf.sin(a), tf.cos(a) # Trigonometry

tf.max(a), tf.min(a) # min max
tf.argmax(a), tf.argmin(a)

tf.ceil(a), tf.floor(a), tf.round(a)
tf.sqrt(a)

tf.cumsum(a) # Cumulative Sum

Finally, if you still feel some NumPy features missing in TensorFlow, you can use this snippet to enable NumPy array methods on tensors.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


print(a.T) # NoAttribute exception

print(tf.transpose(a)) # same functionality

# But I want the a.T notation!!!

from tensorflow.python.ops.numpy_ops import np_config
np_config.enable_numpy_behavior()

print(a.T) # :)

Although all of the snippets above look very similar to NumPy, they are fundamentally very different. The two major differences of tensors and ndarrays are.

Tensors are immutable whereas ndarray is mutable.
Tensor can be placed on various devices (CPU, GPU, TPU, Ram). But ndarray can only be in CPU.

Placing Tensor on GPU

To compute with GPU, you need to copy the tensors into GPU. To do this, first, make sure your TensorFlow is GPU compatible. And then try to create a tensor with tf.device context manager.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18


# You need to build a context with TensorFlow to run statements on a specific device.

# Force execution on CPU
print("On CPU:")
with tf.device("CPU:0"):
 x = tf.random.uniform([1000, 1000])
 assert x.device.endswith("CPU:0")
 print(x.shape)
 print('Done')

# Force execution on GPU #0 if available
if tf.config.list_physical_devices("GPU"):
 print("On GPU:")
 with tf.device("GPU:0"): # Or GPU:1 for the 2nd GPU, GPU:2 for the 3rd etc.
 x = tf.random.uniform([1000, 1000])
 assert x.device.endswith("GPU:0")
 print(x.shape)
 print('Done')

Deep Learning with Keras

Keras is a higher-level API built to work with several deep learning libraries like TF, PyTorch, etc. It became so popular that TF2 started to add this Keras with TF backend inside TF2. So now you do not need to install Keras separately. You can access the Keras API with tensorflow.keras.

Keras has two type of API.

Sequential API: This API is the simplest one for the deep learning model. Here, everything you declare is an object. You instantiate objects and create a Deep Learning model where the output of one layer is the only output of another layer.
Functional API: Keras Sequential API has some limitations. What if you want an output of a layer to be input of two layers? What if you want to concatenate the output of two layers? These are only possible in Functional API.

For this lesson, we will try out the MNIST dataset we used on scikit-learn tutorial. We could use the same load_digit method to load the MNIST dataset with scikit-learn. But TF2 also provides a lot of datasets (more than scikit-learn) to work with. So we will use that dataset in our code.

1
2
3
4
5
6


mnist = tf.keras.datasets.mnist

# The dataset is already split
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)

Dataset shape conventions in Keras

Datasets that are used for a Keras model always have a specific notation for their shape. The shape should always be like (sample_size, sample_shape...). For example, if the size of an image in a dataset is (28, 28) and there are 100 images for an epoch, then the dataset shape will be (100, 28, 28).

Dataset and Input shape is different most of the time. Dataset shape’s first dimension will contain the total number of samples. And Input’s first dimension will contain the number of samples for each iteration. This number is known as batch size. So the Input shape will be (batch_size, 28, 28).

While defining a model architecture both in Sequential API and Functional API, the batch size is automatically inferred while training. So if you want to create a model, the input size should be only (28, 28).

Keras Sequential API

In Keras, a lot of layers for neural networks are implemented. All we need to do to create a neural network is to instantiate those layers, and create a Sequential Model.

So finally, we will create a model that will take an image as input, and output the SoftMax of 10 classes.

The details of the model are shown in this image:

Here is the code to create a Sequential model in Keras.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


import tensorflow as tf

from tensorflow.keras.layers import Dense, Flatten, Conv2D
from tensorflow.keras import Model

model = tf.keras.models.Sequential([
 tf.keras.layers.Flatten(input_shape=(28, 28)),
 tf.keras.layers.Dense(128, activation='relu'),
 tf.keras.layers.Dropout(0.2),
 tf.keras.layers.Dense(10)
])

Keras Functional API

The same model developed above can again be created with Keras Functional API. You may think that of what the purpose is to create the same model in two different APIS. Honestly, there is no benefit to this exercise. But there are many cases where you cannot create a model using only sequential API. We will just create our digit classification model with Functional API just for the sake of introduction to that API.

In the previous model, if you think of the data flow in the model, you will have something like the text below:

1
2
3
4
5
6
7
8
9


(input: 784-dimensional vectors)
 ↧
[Dense (64 units, relu activation)]
 ↧
[Dense (64 units, relu activation)]
 ↧
[Dense (10 units, softmax activation)]
 ↧
(output: logits of a probability distribution over 10 classes)

The functional API works more closely to the data flow than the model architecture. You input a data placeholder into a model, and you will get an output placeholder. You can use that output as an input to other layer instances. Let’s see that in action.

First, we have to instantiate all the layers we need in the model. The layers are not connected to each other in this step.

1
2
3
4
5


# Instantiating the layers. These layers are not connected to each other.
flatten = tf.keras.layers.Flatten()
dense1 = tf.keras.layers.Dense(128, activation='relu')
dropout = tf.keras.layers.Dropout(0.2)
dense2 = tf.keras.layers.Dense(10, activation='softmax')

Then we create an Input placeholder. The shape of the MNIST dataset is (28, 28). You can verify this by seeing x_train.shape.

1

inputs = keras.Input(shape=(28,28))

The final part is to connect all the layers we created so far given a flow of data into those layers. We pass the inputs as a parameter to our first layer and get a return value. Let’s call it x. Then we pass this x to the new layer and get another return value. We will continue this to the last layer. The output of the final layer is the actual output of our model (after activation function).

1
2
3
4
5


inputs = tf.keras.Input(shape=(28,28))
x = flatten(inputs)
x = dense1(x)
x = dropout(x)
outputs = dense2(x)

But like the Sequential API, we do not have any model object. tf.keras.Model objects help you to use a single object to train, modify and predict from different datasets. We can use the constructor of Model to create one. In that model, two parameters are needed. The inputs placeholder, and the outputs placeholder. We can also provide a name for the model.

1

model = tf.keras.Model(inputs=inputs, outputs=outputs, name="mnist_model")

It is always a good practice to create a model inside a function. Then we can create as many models as we want by simply calling that function.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17


def get_model():
 # Instantiating the layers. These layers are not connected to each other.
 flatten = tf.keras.layers.Flatten()
 dense1 = tf.keras.layers.Dense(128, activation='relu')
 dropout = tf.keras.layers.Dropout(0.2)
 dense2 = tf.keras.layers.Dense(10, activation='softmax')

 inputs = tf.keras.Input(shape=(28,28))
 x = flatten(inputs)
 x = dense1(x)
 x = dropout(x)
 outputs = dense2(x)

 model = tf.keras.Model(inputs=inputs, outputs=outputs, name="mnist_model")
 return model

model = get_model()

To see the details of a model, you can use model.summary() which will print all the layers with their output shape, and the number of parameters to train.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


model.summary()

Model: "mnist_model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_8 (InputLayer) [(None, 28, 28)] 0
_________________________________________________________________
flatten_9 (Flatten) (None, 784) 0
_________________________________________________________________
dense_18 (Dense) (None, 128) 100480
_________________________________________________________________
dropout_9 (Dropout) (None, 128) 0
_________________________________________________________________
dense_19 (Dense) (None, 10) 1290
=================================================================
Total params: 101,770
Trainable params: 101,770
Non-trainable params: 0
_________________________________________________________________

Loss functions with Keras

Keras has a lot of loss functions defined inside the tf.keras.losses submodule. As the output is of 10 classes, we will need SparseCategoricalCrossentropy loss function. We can instantiate one like below:

1
2
3
4
5
6


loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)

# For test, the loss_fn can be used like this:
y_true = [1,2]
y_pred = [[0.05, 0.95, 0], [0.1, 0.8, 0.1]]
loss_fn(y_true, y_pred)

Optimizer in Keras

Optimizer is a new word to you till now. Until now, we used a loop to go through the iterations and epochs of the training process. Tensorflow has scikit-learn like .fit and .predict method, so you do not need to explicitly run a loop. Moreover, you do not need to calculate the gradient by yourself or apply the gradient to update the trainable parameters (of course you can do that by yourself). This requires the mathematical knowledge behind the gradient descent algorithms as we did in the last lessons. Also, there are many variants of gradient descent algorithms like Adam, Adadelta, Adagrad, RMSProp, and many more. We have to understand the mathematics behind all of these algorithms to implement them in code.

But TensorFlow has overcome this procedure by introducing tf.keras.optimizers. By default, TensorFlow maintains a graph of calculations of different tensors. That graph has all the tensors as edges and all the operations as nodes. When an operation is done on a tensor, the gradient of the tensor is automatically calculated behind the scene. So, in the backpropagation process, all one needs to do is use learning rate and other algorithms specific parameters to apply that gradient descent algorithm to update weights.

All of these are done using an Optimizer object. Optimizers in TensorFlow take the necessary parameters for that weight optimization algorithm and runs the given number of iterations for you. Let us use the well-established Adam optimizer for now. Check out the link for all other optimization algorithms and how they work as well.

1

optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)

Model Compilation

Now we have all the ingredients to start the training process. We can just call model.fit(x_train, y_train) to start the training! But wait, how would the model know that we will use our initialized Adam optimizer and the loss_fn loss function for the training? We need to link our model with those optimizers and loss functions.

Moreover, while training, we want to watch the loss go down live. Or we may want to see the accuracy increase at each epoch. All these are known as metric. We can optionally provide a metric to our model at compilation to watch it while training.

1
2
3
4
5


model.compile(
 optimizer=optimizer,
 loss=loss_fn,
 metrics=['accuracy']
)

Training loop

As stated earlier, we do not need to explicitly run a loop for the training. The optimizer will handle it for us. Like scikit-learn, we can run model.fit to see the training live.

1

model.fit(x_train, y_train, epochs=5)

Model evaluation

To see how good our model is on the cross-validation set, we can run an evaluation of our model on the cross-validation set.

1

model.evaluate(x_test, y_test)

We can also predict new datasets using the trained model. If we have a new dataset names x_test_new, we can get y_pred using the predict method.

1

y_pred = model.predict(y_test_new)

Callback functions, Checkpoint, saving and loading model

Training a large neural network is a long time process. Many current neural networks even take weeks to fully train. Imagine you are training a model, but almost at the end of the training, you have a power-cut or some other problem. You will need to retrain the whole model again. The training of the model can be saved at different points using a ModelCheckpoint callable callback object.

While training a model, TensorFlow provides a way to call any callable (functions, classes, etc.) at each epoch. TF2 also provides a lot of already implemented callables. You can use those callables by passing them to the .fit method. In the below snippet, we will use the tf.keras.callbacks.ModelCheckpoint callback to save the model at each epoch if the metric is higher than the current save.

1
2
3
4
5
6


model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
 filepath='checkpoint/',
 save_weights_only=True,
 monitor='accuracy',
 mode='max',
 save_best_only=True)

Then we can pass this callback while training the model.

1

model.fit(x_train, y_train, epochs=5, callbacks=[model_checkpoint_callback])

If you want to resume the training on a new machine after an interruption, just load the model weights before training.

1

model.load_weights('checkpoint/')

While training the model, you will find a new folder named checkpoint where the model with the best accuracy will be saved.

After training the model, you can save and load the model object to a file and share that with anyone. For this, you need to install some dependency libraries. Install them with the following commands:

1

pip install -q pyyaml h5py

Now use this snippet to save and load model

1
2
3
4
5


# Save the model
model.save('saved_model/my_model')

# Load the model
model = tf.keras.models.load_model('saved_model/my_model')

Conclusion

In this lesson, we have gone through all the fundamentals of TensorFlow 2. But honestly, we just scratched the surface. TensorFlow library is huge. It is used both by many researchers in Academia and by many real-life programmers for production-ready software. Now that you can load and preprocess the dataset, build a model, train a model and finally save the model; go ahead and look for some dataset online. Then train a new model on that dataset and see what the results are. In another lesson, we will discuss how to effectively find a dataset and cite that dataset in your ML model. We will soon also look at many popular machine learning algorithms like CNN, LSTM, and many established models (e.g. Yolo, R-CNN, GPT3, etc.). Until then, make sure you are very comfortable working with TensorFlow.

Machine Learning Tutorial - Lesson 10

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Deep Learning and Computer Vision

Introduction

In the previous lesson, you were introduced to one of the most popular frameworks for Deep Learning, TensorFlow. In this lesson, we will use TensorFlow to create some popular deep neural networks. In the meantime, you will also learn which deep learning technology should be used with what kind of data, and how you can customize a deep neural network to your needs. So let us get started.

Neural Network for Image Data

First, we will start with Image datasets. There is a whole different field for images, videos, and other visual data (LIDAR, X-Ray, Depth Camera, etc.) called Computer Vision. One of the most compelling types of AI is computer vision which you’ve almost surely experienced in any number of ways without even knowing. Some of the examples are:

You have used face filters in Snapchat, Facebook Messenger, YouCam Perfect, etc. apps, right?
For security purposes, sometimes an object from a video or image is blurred/removed/pixelated. With CV, you can do that efficiently.
You might have used google translate’s image to text feature. It takes an image and reads the text within it.
There are many examples of image classification. For example, many machines use image processing to classify different coins and notes.

All of the above applications of Computer Vision are done using Deep Learning. That is why, nowadays, you cannot imagine Image processing without deep learning.

To get started with deep learning, we need to have a look at how images are stored in a computer.

Structure of an Image

There are many formats of images (png, jpg, BMP, tiff, etc.). But when an image is loaded for processing in the memory, trivially it is loaded as a 3-dimensional tensor. So images are tensor objects whose dimension is always 3.

Among these three dimensions, the first two dimensions are for the height and width of the image. These only two dimensions together form a matrix that represents a channel of an image. Mostly, this is a color channel. The third dimension represents the number of channels of the image. For example, if the shape of an image is (500, 600, 3), that means the image is 500 pixels high, 600 pixels wide, and has 3 color channels.

Depending on the number and order of color channels, images can be in different color formats. Some examples of these formats are given below:

RGB: This is the simplest and most popular format. Images have 3 color channels, represents Red, Green, and Blue (RGB). Keep the order in mind because in RGB, always the first channel is for the color Red, the second is for Green and the third is for Blue.
RGBA: If you add an alpha channel to the RGB image, then you get RGBA images. These images have opacity. PNG image formats are compatible with RGBA image format as they also have this opacity channel.
BGR: Blue, Green, and Red channels in the image. This is the same as RGB but the order of the channel is different. Although this format is not that popular, for some reason the most popular computer vision library OpenCV used this format as default.
Grayscale: There is only one channel of the image. You can represent it in both (h,w,1) or just (h,w) format.

All the above formats consist of pixels/color values which can be represented again in two ways.

In computer vision, there are also other data formats. For example, there is a voxel (Volumetric Pixel). This format is a four-dimensional tensor of share $(h,w,d,c)$, where they represent the height, width, depth, and channel respectively. There are also Point Cloud representations for a LIDAR dataset. As we are covering the basics, we will only stick to the formats mentioned above.

Let us define an image by ourselves. In the below example, we will create a gradient grayscale image and display it.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12


import numpy as np
import matplotlib.pyplot as plt

def get_gradient_2d(start, stop, width, height, is_horizontal):
 if is_horizontal:
 return np.tile(np.linspace(start, stop, width), (height, 1))
 else:
 return np.tile(np.linspace(start, stop, height), (width, 1)).T

img = get_gradient_2d(0, 255, 500, 500, True) # shape = (500, 500)

plt.imshow(img, cmap="gray")

And here is the result:

OpenCV

Image processing and Computer Vision are vast fields of study in both academics and industry. It is simply infeasible for anyone to use primitive libraries like NumPy for image manipulation. For this, many specialized image processing libraries are created. Among them, OpenCV is by far the most popular Image processing library. It is fast, written in C++, ported to many programming languages like Java, JS, Python, C#, etc. It can not only processing images but also read different image formats, convert between different formats, read/write to different kinds of source and sink devices like webcam, video file, image file, etc. The world of image processing and computer vision is just incomplete without this amazing library.

As I said earlier, the default image format for OpenCV is BGR. So to work with all other libraries, you need to convert the image to RGB or RGBA first. Let us see an example where we read an image and display it with matplotlib. You can download the famous image processing sample picture of Leena from here.

1
2
3
4
5


import cv2
img = cv2.imread('leena.png')
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
print(img.shape)
plt.imshow(img)

If you run the code, you will see that the image is of shape (463,453,3). The result will look something like this.

Did you notice the integer flag named cv2.COLOR_RGB2BGR while converting an image? Get used to it, OpenCV uses hundreds of integer flags like that. You need to use those flags as parameters of its methods.

Image Preprocessing

You already know that, in machine learning, the first step after loading data is preprocessing. In python, preprocessing images is mostly done with NumPy and OpenCV. In this lesson, we will focus on the preprocessing parts that are needed for feeding it into a neural network. Below are the four things you have to do before feeding images into your network.

Resizing: All neural networks have to take in the same tensor shapes. For this, you need to find the right size for all your images and then resize all images into that size. You can use the cv2.resize(img, shape) method for resizing. For image resizing, you also need to know about different interpolation techniques. The most common one used is cv2.INTER_NEAREST which can be passed to the function while resizing.
Intensity Scaling: Each pixel value of images is in a range from 0 to 255 most of the time. But to get the best result, it is proved that the range of pixel values should be between 0 and 1, or -1 and 1. You can use the skimage.exposure.rescale_intensity(img) method from scikit-image module to rescale images to any range. The background is the same as rescaling any attribute in a dataset which we did previously.
Quantizing: In your given images are not photos of scenarios with a wide range of colors, quantizing your images and just inputting only 5-10 color values instead of 255 will drastically improve your model’s prediction speed. Besides, for certain types of images (region of interest of object detection for example) this method will also increase the accuracy of your model. You need to do this manually using NumPy.
Thresholding: If you only need a certain color or range of color, then you can apply a threshold to that range of color in your image. There are many kinds of thresholds’ that you can do. First, let us look at some simple threshold methods OpenCV offers us. Each threshold method is briefly described above the method in comments. We will work with the gradient image we created with NumPy.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18


img = cv.imread('gradient.png',0)
# All pixels above 127 will be 1, others will be 0
ret,thresh1 = cv.threshold(img,127,255,cv.THRESH_BINARY)
# All pixels above 127 will be 0, others will be 1
ret,thresh2 = cv.threshold(img,127,255,cv.THRESH_BINARY_INV)
# Make all pixels greater that the value 127 to the value 255
ret,thresh3 = cv.threshold(img,127,255,cv.THRESH_TRUNC)
# Make all pixels greater that the value 127 to the value 0
ret,thresh4 = cv.threshold(img,127,255,cv.THRESH_TOZERO)
# Make all pixels less that the value 127 to the value 0
ret,thresh5 = cv.threshold(img,127,255,cv.THRESH_TOZERO_INV)
titles = ['Original Image','BINARY','BINARY_INV','TRUNC','TOZERO','TOZERO_INV']
images = [img, thresh1, thresh2, thresh3, thresh4, thresh5]
for i in range(6):
 plt.subplot(2,3,i+1),plt.imshow(images[i],'gray',vmin=0,vmax=255)
 plt.title(titles[i])
 plt.xticks([]),plt.yticks([])
plt.show()

The result of the code is below.

These threshold algorithms have a fixed function, that is applied to all the pixels, regardless of the value of that pixel or its neighborhood. These kinds of threshold algorithms are call global threshold algorithms. There are also local/adaptive threshold algorithms. These adaptive threshold algorithms take a neighborhood into consideration and then apply threshold to some pixels. We will discuss only one threshold algorithms, adaptive mean threshold. This algorithm takes an kernel sized neighborhood and takes the mean of that kernel as the thresholding value. Then it applies the thresholding on that region. OpenCV has cv2.adaptiveThreshold function for this. Let us see what it does in realtime:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13


img = cv2.imread('sudoku.jpg',0)
img = cv2.medianBlur(img,5)
ret,th1 = cv2.threshold(img,127,255,cv2.THRESH_BINARY)
th2 = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
 cv2.THRESH_BINARY,11,2)
titles = ['Original Image', 'Global Thresholding (v = 127)',
 'Adaptive Mean Thresholding']
images = [img, th1, th2]
for i in range(3):
 plt.subplot(2,2,i+1),plt.imshow(images[i],'gray')
 plt.title(titles[i])
 plt.xticks([]),plt.yticks([])
plt.show()

Finally, we will discuss the last and most popular threshold algorithm. You do not have to select a threshold value (like we did 127 for all the above examples). The algorithm will select the optimal threshold value which will divide the whole image into the best binary regions. It is OTSU’s binary threshold algorithm. Before understanding this algorithm, you need to learn about the histogram of images.

Histogram of Images

Each pixel of images has a range of values. It can be a discrete/quantized number of values: for example integer values from 0 to 255. They can also be continuous values ranging from 0.0 to 1.0. Either way, you need to plot a graph of the distribution of pixel values of the image using a 2D line plot. The X-axis will represent the range of the values. So it can be in between 0-255 or 0-1. The Y-axis represents the number of pixels for the corresponding pixel value in the X-axis. For example, if an image has a total of 400 pixels whose values are 55, then the Y value of the plot at $X=55$ is $400$. So the maximum value of Y-Axis would be $hw$ where $h$ is height and $w$ is the width of the image.

Matplotlib provides us utility function to get the histogram of an image. Let’s see how the histogram of a binary image looks like.

1
2
3


img = cv2.imread('ex.jpg',0)
plt.hist(img.ravel(),256,[0,256])
plt.show()

As you can see, most of the pixels’ color values are around 100.

OTSU’s threshold algorithm uses this histogram information to calculate the optimal value to threshold an image. Otsu’s algorithm tries to find a threshold value (t) that minimizes the weighted within-class variance. We will not get into the mathematical details of this algorithm (it is quite interesting to look at though). OpenCV provides cv2.threshold(img,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) function that can be used for this. There are many online documentations to look for on the internet. Sometimes, applying a blurring (e.g. Gaussian Filter) will make the result of the threshold algorithm much better.

For preprocessing, you can also apply histogram equalization to make the color distribution of all the images very similar. It is appropriate in the case of digit classification or character recognition. I will cover all these image processing techniques in a later lesson.

Image Classification with Deep Learning

Now we are at the heart of today’s lesson. We will use 2 types of neural networks on the popular Fashion MNIST dataset to classify 10 different fashion images. In the previous lesson, I presented you with how to easily create a neural network using TensorFlow within just 10 lines of code. We use that same code to create a digit classifier in this lesson.

Although I will only show you an image classification example, there are many other principle tasks for deep learning. The most common types are:

A. For single Object

Image classification: Is this object in the photo?
Image classification: What object (among some predefined) is in the photo?
Object localization: Where is this object located in the image?
Object segmentation: Just select those pixels which are part of an object in the image.

B. For multiple objects in the same image:

Object detection: Where each object is located in a single image?
Instance Segmentation: Classify all pixels of an image to mark it as a part of some object (or background).

A famous image for understanding these types is shown by Mike Tamir.

Linear Neural Network

Linear neural networks are built with neurons where each neuron is responsible to select/determine if a feature in that value is present. When you add a lot of neurons to a single layer, it tries to detect specific features in the whole dataset that will help you to achieve the final goals. The first layer of the network is only able to detect simple features in the data. Later layers take in the detected features from the previous layers and detect more complex features among them.

For images, each neuron will take in all the pixels at once, and try to predict a number. That prediction number can mean anything (mostly unrecognizable by humans). Next, that value will be again propagated to all the neurons on the next layer. For classification, the output layer will have only 10 neurons. Each neuron will be responsible to predict a value that tells us how much is it likely, that the image is $n \n {0…9}$.

Images are a tensor of 3 dimensions. MNIST images are grayscale images, so they are actually of 2 dimensions (So the whole dataset is of shape (8000, 28, 28)). But Linear neural networks only take a dataset with a linear feature set. Each feature can contain only 1 value for sample data. So we need to convert each of our images into a linear example. We can do this pretty easily by flattening each image using the NumPy flatten method. But then we need to loop through all the images and call the method on each of them. Instead, we can just use the np.reshape method to reshape the whole dataset into the desired shape.

Before that, lets import all the libraries we need for this:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15


# Basic
import pandas as pd
import numpy as np

# for visualization
import matplotlib.pyplot as plt

# Creating the model
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier

# To Evaluate the models
from sklearn.model_selection import cross_val_score, cross_validate

Next, we load the dataset into memory. We will use TensorFlow’s dataset submodule to load the dataset. It will download the dataset from the internet and return it to a python variable.

1
2


mnist = keras.datasets.fashion_mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()

After loading the dataset, we will reshape the dataset into our desired linear shape. We might not know the number of samples for the data, so we can use the shape of the NumPy dataset to get the number of examples in both of our training and testing sets. After reshaping the dataset, we will do a very simple preprocessing. The range of color of the pixel is 0-255. We will divide it by 255 so that we can get a range from 0-1. This is necessary for most deep neural networks or there might be exploding gradient problem.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


# Reshape each iamges from 2D to 1D array.
# So the whole dataset is now 2D
X_train = X_train.reshape(X_train.shape[0], 784)
X_test = X_test.reshape(X_test.shape[0], 784)

y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

X_train = X_train / 255.0
X_test = X_test / 255.0

Now that our data is ready and preprocessed, let’s have a peek at some of the images we are dealing with. Data visualization is very important for any model development as it gives you insights into different approaches you can take with your data and model. In this dataset, here each class label [0..9] represents a cloth/shoe in the dataset.

1
2
3
4
5
6
7


fig, axes = plt.subplots(nrows=2, ncols=5,figsize=(15,5))
ax = axes.ravel()
for i in range(10):
 ax[i].imshow(X_train[i].reshape(28,28), cmap="gray")
 ax[i].title.set_text('Class: ' + str(np.argmax(y_train, axis=1)[i]))
plt.subplots_adjust(hspace=0.5)
plt.show()

Now we will build our Artificial Neural Network (ANN). The model will consist of only 1 hidden layer (because the dataset is very simple). We will use uniform weight initialization, categorical cross-entropy for loss, and Adam optimizer.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14


# Create a model (we will use Sequential API)
model = Sequential()
# Input layer
model.add(layers.InputLayer(X_train.shape[1]))
# Hidden layers
model.add(layers.Dense(128, kernel_initializer='uniform', activation='relu'))
# Output layer
model.add(layers.Dense(10, kernel_initializer='uniform', activation='softmax'))

# Add accuracy as metric and categorial crossentropy as loss, optimizer adam
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])


model.summary()

The model summary output is:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 128) 100480
_________________________________________________________________
dense_1 (Dense) (None, 10) 1290
=================================================================
Total params: 101,770
Trainable params: 101,770
Non-trainable params: 0

Finally, we train the model with the X_train and label y_train.

1

model.fit(X_train, y_train, epochs=10)

We will see the training progress at each epoch. After training the model, we can evaluate the model on the test set.

1

model.evaluate(X_test, y_test)

The result is 87% accuracy. Keep in mind that if you tune the model, preprocess the data properly, it is possible to get 90+ accuracy with just a linear neural network. And 99.99% accuracy is possible if you train these image datasets into a well-developed Convolutional Neural Network.

Convolutional Neural Network

Convolution Operation

Convolutional Neural Network is the most efficient neural network developed for images until now. There are a lot of variations of CNN out there. In this lesson, I will try my best to make CNN as much clarity as possible. The first thing to understand for CNN is how convolution operation works.

In purely mathematical terms, convolution is a function derived from two given functions by integration which expresses how the shape of one is modified by the other. That can sound baffling as it is, but to make matters worse, we can take a look at the convolution formula:

$$(f*g)(t) = \int_{-\infty}^{\infty} f(\tau)g(t-\tau) d\tau$$

So let us not think about the mathematical background, and see a practical example.

The convolution operation is done on an image (for this context) with a kernel. Most of the time, the kernel is much smaller in size than the image. Suppose the image is a (5x5) matrix, and the kernel is a (3x3) matrix.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13


A = [
 [1,2,3,4,5],
 [5,4,3,2,1],
 [6,7,8,9,8],
 [3,4,5,6,7],
 [5,4,3,2,1],
]

k = [
 [1,2,3],
 [3,2,1],
 [4,5,6],
]

The convolution between these two will result in an image of shape (4,4). The kernel will start from the top left corner of the image (where it can fit inside the image). Then it will element-wise multiply and sum up all to get a single value.

1
2
3
4
5
6
7


[
 [1x1 + 2x2 + 3x3] +
 [5x3 + 4x2 + 3x1] +
 [6x4 + 7x5 + 8x6] +
]

= 147

Then the kernel will again shift to the right by 1 and again do the same operation to the submatrix and the kernel.

1
2
3
4
5
6
7


[
 [2x1 + 3x2 + 4x3] +
 [4x3 + 3x2 + 2x1] +
 [7x4 + 8x5 + 9x6] +
]

= 162

This process will continue until a place where the kernel does no more fit in the image. Then the kernel will start over from the left of the image, going down 1 pixel. So after the first row, the first result of the second row will look like this:

1
2
3
4
5
6
7


[
 [5x1 + 4x2 + 3x3] +
 [6x3 + 7x2 + 8x1] +
 [3x4 + 4x5 + 5x6] +
]

= 124

This will continue until the end, and the resulting image will be the summing value put in the center of the kernel while the multiplication operation was being done. This will result in a (4x4) matrix. As you always have to select the center location of the kernel to place a pixel value, kernel shape cannot be even size.

1
2
3
4
5


A * k = [
 [147, 162, 165],
 [124, 139, 152],
 [124, 121, 112],
]

Using OpenCV, you can apply the convolution operation using filter2D(img, -1, kernel) function. But OpenCV tries to keep the output the same size as input. To keep this same shape, it first adds a border to the image so that after applying the convolution, the result is of the same shape as the input image. There are many cv2.BOARDER* flags that you can use to specify different ways to pad the image. We will just ignore them for now and only take the image as output:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15


A = np.array([
 [1,2,3,4,5],
 [5,4,3,2,1],
 [6,7,8,9,8],
 [3,4,5,6,7],
 [5,4,3,2,1],
], dtype='uint8')

k = np.array([
 [1,2,3],
 [3,2,1],
 [4,5,6],
], dtype='uint8')

cv2.filter2D(A, -1, kernel=k)[1:-1, 1:-1]

The resulting image is:

1
2
3


array([[147, 162, 165],
 [124, 139, 152],
 [124, 121, 112]], dtype=uint8)

CNN Layer

CNN is a neural network based on 2D convolution operation. The values in the kernel are considered as weights. You can select a kernel initialization method (uniform, he_uniform, etc.) and the size of the kernel. At each training iteration, the weights of the kernels will be updated with the loss and gradient in the backpropagation step.

In TensorFlow, the code will not be different from that of the linear neural network. The Dense layers of the model definition will only be replaced with Conv2D layers. Let’s see the whole code at once:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44


# Basic
import pandas as pd
import numpy as np
import tensorflow as tf

# for visualization
import matplotlib.pyplot as plt

# For preprocessing the data
from tensorflow.keras.utils import to_categorical

# Creating the model
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers


mnist = tf.keras.datasets.fashion_mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()

# We do not need to reshape the images into a linear datashape. 
# CNN takes an image as input with different number of channels you define
# These are grayscale images, so the number of channels is one
X_train = X_train.reshape(X_train.shape[0], 28,28,1)
X_test = X_test.reshape(X_test.shape[0], 28,28,1)

y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

X_train = X_train / 255.0
X_test = X_test / 255.0

model = Sequential()
model.add(layers.InputLayer((28,28,1)))
model.add(layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='uniform'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(100, activation='relu', kernel_initializer='uniform'))
model.add(layers.Dense(10, activation='softmax'))
# compile model
opt = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

model.fit(X_train, y_train, epochs=10)
model.evaluate(X_test, y_test)

If you have a GPU (and you properly installed TensorFlow, CUDA, and everything else to use GPU following our previous tutorial), then this will take less than a minute. But if you are running TensorFlow on CPU, you might want to grab a coffee because it will take about 4-5 minutes.

The evaluation result is 90%. This is higher than a linear neural network. All the other hyperparameters other than the model architecture are the same. So you can see that for image operations, CNN is better than trivial ANN.

More Neural Networks: How to explore them?

People worry that computers will get too smart and take over the world, but the real problem is that they’re too stupid and they’ve already taken over the world

Pedro Domingos

Developing a machine learning model is just a part of Art. There are no hard rules on how to develop the perfect machine learning model. You just have to be experienced enough to have a good feeling that “these” kinds of models should work well on these kinds of data. One can create a very wide neural network for a task, and another person can create a very deep and narrow neural network. None of them are guaranteed to work perfectly and any of them can fail training properly.

So you need a lot of experience to create good neural network architectures. You will need to see and run models developed by other experts out there. There are many websites where you can find different state-of-the-art neural network models for different kinds of data. I am listing below some of the most popular ones you want to look at.

ModelZoo.com: https://modelzoo.co/
ONNX Model Zoo: https://github.com/onnx/models
Keras Applications: https://keras.io/api/applications/
IBM exchange model repository: https://developer.ibm.com/exchanges/models/
Computer Vision Model Library: https://models.roboflow.com/

It seems that all the major deep learning framework owners have their deep learning model repositories.

TensorFlow: https://github.com/tensorflow/models
PyTorch: https://github.com/Cadene/pretrained-models.pytorch
caffe: https://github.com/BVLC/caffe/wiki/Model-Zoo
caffe2: https://github.com/caffe2/caffe2/wiki/Model-Zoo
Lasagne: https://github.com/Lasagne/Recipes

I think these will be enough, for now, to try some machine learning models by yourself. Many of these are pre-trained models, so you only need to use them for prediction. But you may also want to train a model with a dataset. Let us discuss where to find a dataset that you can use in your own software.

Dataset: Where to find them?

The best place to find a dataset is Kaggle-Datasets. Besides kaggle, you can look at the following websites to search for a dataset:

UCI Machine Learning Repository: https://archive.ics.uci.edu/ml/
Awesome Public Dataset: https://github.com/awesomedata/awesome-public-datasets
Microsoft Research Open Data: https://msropendata.com/
Amazon Open Data Registry: https://registry.opendata.aws/
Visual Data Discovery: https://www.visualdata.io/discovery

If you do not want to browse these repositories, you can use Google to do it for that. But not the google.com. It is Google Dataset Search Engine

There are also government-sponsored public datasets for many countries. Some of the popular ones are:

US Gov Data: https://www.data.gov/
Indian Government Dataset Repository: https://data.gov.in/
European Government Datasets: https://data.europa.eu/euodp/data/dataset
New Zealand’s Government Dataset: https://catalogue.data.govt.nz/dataset
Northern Ireland Public Dataset: https://www.opendatani.gov.uk/

Tell us in the discussion section if your country has a dataset repository.

Conclusion

This is the end of the deep learning on image dataset lesson. In this lesson, I tried to mention all you need to know about computer vision and deep learning. I did not go thoroughly into any topic because most of the libraries are open source and you can get to their official tutorials which will be better than any other place. This will also help you to learn to read official documentation.

Machine Learning Tutorial - Lesson 11

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Real World Deep Neural Network Examples

Introduction

After the last lesson, I hope you have got the primer of how Deep Learning and TensorFlow work. In this lesson, we will use some real-life Deep Learning networks, that are used in production for many different applications. The work needed behind creating and training these models is a lot, and you simply cannot go through all the details of each model in this single lesson. So we will focus more on the application and specialty of these models. We will only load the pre-trained models, and run them on some of our custom images.

Traditional Neural Networks

AlexNET

AlexNet CNN is probably one of the simplest methods to approach understanding deep learning concepts and techniques. This is why this network is the first one I want to introduce to all of you. AlexNet is not a complicated architecture when it is compared with some state-of-the-art CNN architectures that have emerged in more recent years.

Note: We will create a complete Keras pipeline for only this section. As most of the pipelines in Keras are almost the same, the later discussion will only go through the model architecture definition function.

Maybe you are already tired to see the fashion MNIST and IRIS dataset. That’s why we will use a different dataset in this lesson, CIFAR-10 dataset. This can also be found inside the datasets submodule of TensorFlow.

But first things first, we import all the modules we need for the pipeline:

1
2
3
4
5


import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
import matplotlib.pyplot as plt

And now we load the training set. CIFAR-10 is a dataset of images of different animals just like the MNIST dataset. But the dataset is larger than MNIST. It contains 50 thousand images for the training set and 10 thousand images for the test set. We will divide the training dataset into two again: for training and validation. The first five thousand images will be for validation, and the later ones will be for training. We can easily do this using NumPy slicing.

1
2
3
4
5


(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.cifar10.load_data()
CLASS_NAMES= ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']

validation_images, validation_labels = train_images[:5000], train_labels[:5000]
train_images, train_labels = train_images[5000:], train_labels[5000:]

The most convenient way to train a dataset is through a TensorFlow dataset object as discussed in a previous lesson. So we will convert these NumPy arrays into a TensorFlow dataset using from_tensor_slices method of the Dataset submodule. Finally, we can see the number of images in each division using the cardinality method.

1
2
3
4
5
6
7
8


train_ds = tf.data.Dataset.from_tensor_slices((train_images, train_labels))
test_ds = tf.data.Dataset.from_tensor_slices((test_images, test_labels))
validation_ds = tf.data.Dataset.from_tensor_slices((validation_images, validation_labels))

get_ds_size = lambda ds: tf.data.experimental.cardinality(ds).numpy()
print("Training data size:", get_ds_size(train_ds))
print("Test data size:", get_ds_size(test_ds))
print("Validation data size:", get_ds_size(validation_ds))

The output is:

1
2
3


Training data size: 45000
Test data size: 10000
Validation data size: 5000

Let us take a look at some of the images using matplotlib. We will use the take method to randomly sample some images and display them in subplots.

1
2
3
4
5
6


plt.figure(figsize=(20,20))
for i, (image, label) in enumerate(train_ds.take(5)):
 ax = plt.subplot(5,5,i+1)
 plt.imshow(image)
 plt.title(CLASS_NAMES[label.numpy()[0]])
 plt.axis('off')

Now comes the preprocessing part. The pixel values of the images are now in the range of 0-255. We need to convert it into a range of -1 to 1. Thanks to the convenience of the Dataset submodule, you can use the per_image_standardization method on the dataset. After resizing it to the model’s desired shape, we shuffle the data and apply a batch size of 32 to train the model.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20


def preprocess(image, label):
 # Standardize each images
 image = tf.image.per_image_standardization(image)
 # AlexNET takes input of size (227,227)
 # Resize all images to 277x277
 image = tf.image.resize(image, (227,227))
 return image, label

train_ds = (train_ds
 .map(preprocess)
 .shuffle(buffer_size=train_ds_size)
 .batch(batch_size=32, drop_remainder=True))
test_ds = (test_ds
 .map(preprocess)
 .shuffle(buffer_size=train_ds_size)
 .batch(batch_size=32, drop_remainder=True))
validation_ds = (validation_ds
 .map(preprocess)
 .shuffle(buffer_size=train_ds_size)
 .batch(batch_size=32, drop_remainder=True))

The model architecture contains a set of Convolution-BatchNormalization-MaxPool sets first. Then finally the data is flattened and two fully connected layers of 4096 units are used. As the output is the probability of 10 classes, the final fully connected layer will have 10 units only.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28


model = Sequential([
 layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),
 layers.BatchNormalization(),
 layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),

 layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding="same"),
 layers.BatchNormalization(),
 layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),

 layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding="same"),
 layers.BatchNormalization(),

 layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding="same"),
 layers.BatchNormalization(),

 layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding="same"),
 layers.BatchNormalization(),
 layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),

 layers.Flatten(),
 layers.Dense(4096, activation='relu'),
 layers.Dropout(0.5),

 layers.Dense(4096, activation='relu'),
 layers.Dropout(0.5),
 layers.Dense(10, activation='softmax')
])
model.summary()

For training the model, we can use both Categorical Crossentropy or Sparse Categorical CrossEntropy as the loss function. But using only Categorical cross-entropy will lead us to another step (converting the labels to one-hot encoding). So we will use the sparse one as loss function. The optimizer is the Stochastic Gradient Descent algorithm.

1

model.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(learning_rate=0.001), metrics=['accuracy'])

We can now start the training with the fit method on the model.

1
2
3
4
5
6


model.fit(
 train_ds,
 epochs=50,
 validation_data=validation_ds,
 validation_freq=1,
)

Note: This will take a lot of time depending on the machine you are running on. Also if you are low on available memory in your computer, your system may get stuck. Alternatively, you can run a notebook on Google Colab or Kaggle using GPU for free.

To evaluate the model, we can run the evaluate method on the test_ds set.

1

np.mean(model.evaluate(test_ds))

1
2


312/312 [==============================] - 8s 27ms/step - loss: 0.9814 - accuracy: 0.7439
0.8626266404836566

So the mean accuracy on the test set is 86.26%. That is actually good for this old neural network created in 2012.

You Only Look Once (YOLO)

The first model we will try is the YOLO object detection model. This is a model where you can give the image and it will detect a lot of objects for you. The first Yolo model (Yolo V1) could detect 80 real-world objects. YOLO is a state-of-the-art, real-time object detection system. On a Pascal Titan X, it processes images at 30 FPS and has an mAP of 57.9% on COCO test-dev.

The major advantage of YOLO is that it is very fast. Unlike other Classifier based neural networks, YOLO applies image classification on multiple regions in parallel with the help of GPU, and if an object is detected, it uses an object classifier to classify it.

There is a lot of implementation out there for YOLO. The original code for darknet can be used with OpenCV’s DNN module. But we will use a much simpler version by Anushka Dhiman. Although I have no affiliation with any of the persons’ repositories, I only select these because I found these the easiest to use. Her repository supports TensorFlow 2 and is up-to-date with Yolo v3. Let us first download the code and set up the environment.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


git clone https://github.com/anushkadhiman/YOLOv3-TensorFlow-2.x.git
cd YOLOv3-TensorFlow-2.x

pip install -r ./requirements.txt

# yolov3
wget -P model_data https://pjreddie.com/media/files/yolov3.weights

# yolov3-tiny
wget -P model_data https://pjreddie.com/media/files/yolov3-tiny.weights

There are two YOLO models: yolov3 is the large model with the highest accuracy, and yolov3-tiny is designed to have higher FPS but a little less accuracy. We will try the regular yolov3. To use this on an image, there is a function names detect_image in the utils file. All we need to do is load the model with Load_Yolo_model and then use that function on the image.

1
2
3
4
5
6
7


import cv2
from yolov3.utils import detect_image, detect_realtime, detect_video, Load_Yolo_model, detect_video_realtime_mp
from yolov3.configs import YOLO_INPUT_SIZE

image_path = "your_image.jpg"
yolo = Load_Yolo_model()
detect_image(yolo, image_path, "detect.jpg", input_size=YOLO_INPUT_SIZE, show=True, rectangle_colors=(255,0,0))

If you want to run the object detection on a video file, then you can use detect_video function.

1

detect_video(yolo, your_video_path, "", input_size=YOLO_INPUT_SIZE, show=False, rectangle_colors=(255,0,0))

Finally, you can also use your primary webcam to apply object detection live.

1

detect_realtime(yolo, '', input_size=YOLO_INPUT_SIZE, show=True, rectangle_colors=(255, 0, 0))

Let us now get a little deep into the implementation of the neural network. The model architecture in python is given below. Yes, it is huge. Most of the established deep learning networks are like this.

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151


from keras.layers import Conv2D
from keras.layers import Input
from keras.layers import BatchNormalization
from keras.layers import LeakyReLU
from keras.layers import ZeroPadding2D
from keras.layers import UpSampling2D
from keras.layers.merge import add, concatenate
from keras.models import Model

def convolution_block(x, convs, skip=True):
 count = 0
 for conv in convs:
 if count == (len(convs) - 2) and skip:
 skip_connection = x
 count += 1
 # The model is designed with only left and top padding
 if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x)
 x = Conv2D(conv['filter'],
 conv['kernel'],
 strides=conv['stride'],
 padding='valid' if conv['stride'] > 1 else 'same',
 name='conv_' + str(conv['layer_idx']),
 use_bias=False if conv['bnorm'] else True)(x)
 if conv['bnorm']:
 x = BatchNormalization(
 epsilon=0.001,
 name='bnorm_' + str(conv['layer_idx'])
 )(x)
 if conv['leaky']:
 x = LeakyReLU(
 alpha=0.1,
 name='leaky_' + str(conv['layer_idx'])
 )(x)
 return add([skip_connection, x]) if skip else x

def make_yolov3_model():
 input_image = Input(shape=(None, None, 3))
 # Layer 0 => 4
 x = convolution_block(input_image, [
 {'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0},
 {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1},
 {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2},
 {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}
 ])
 # Layer 5 => 8
 x = convolution_block(x, [
 {'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5},
 {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6},
 {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}
 ])
 # Layer 9 => 11
 x = convolution_block(x, [
 {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9},
 {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}
 ])
 # Layer 12 => 15
 x = convolution_block(x, [
 {'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12},
 {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13},
 {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}
 ])
 # Layer 16 => 36
 for i in range(7):
 x = convolution_block(x, [
 {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3},
 {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}
 ])
 # There is a skip from this layer to the last layer. So storing x
 skip_36 = x
 # Layer 37 => 40
 x = convolution_block(x, [
 {'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37},
 {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38},
 {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}
 ])
 # Layer 41 => 61
 for i in range(7):
 x = convolution_block(x, [
 {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3},
 {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}
 ])
 skip_61 = x
 # Layer 62 => 65
 x = convolution_block(x, [
 {'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62},
 {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63},
 {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}
 ])
 # Layer 66 => 74
 for i in range(3):
 x = convolution_block(x, [
 {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3},
 {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}
 ])
 # Layer 75 => 79
 x = convolution_block(x, [
 {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75},
 {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76},
 {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77},
 {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78},
 {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}
 ], skip=False)
 # Layer 80 => 82
 yolo_82 = convolution_block(x, [
 {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 80},
 {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}
 ], skip=False)
 # Layer 83 => 86
 x = convolution_block(x, [
 {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}
 ], skip=False)

 x = UpSampling2D(2)(x)
 x = concatenate([x, skip_61])

 # Layer 87 => 91
 x = convolution_block(x, [
 {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87},
 {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88},
 {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89},
 {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90},
 {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}
 ], skip=False)

 # Layer 92 => 94
 yolo_94 = convolution_block(x, [
 {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 92},
 {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}
 ], skip=False)

 # Layer 95 => 98
 x = convolution_block(x, [
 {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 96}
 ], skip=False)

 x = UpSampling2D(2)(x)
 x = concatenate([x, skip_36])

 # Layer 99 => 106
 yolo_106 = convolution_block(x, [
 {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 99},
 {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 100},
 {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 101},
 {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 102},
 {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 103},
 {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 104},
 {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}
 ], skip=False)

 model = Model(input_image, [yolo_82, yolo_94, yolo_106])
 return model

Training the model will literally take days if you are planing are planning to train it on a large dataset like Microsoft COCO. There is a total of 62001757 parameters you need to pass through when feeding forward.

VGG Network

Named after Visual Geometry Group, Oxford University, the VGG network is based on repeatable blocks named VGG-blocks. These are groups of convolutional layers that use small filters and max pooling layers. Assuming that you already can readout a code and understand the neural network architecture, I am giving the python code for VGG_Block without describing it:

1
2
3
4
5


def vgg_block(layer_in, n_filters, n_conv):
 for _ in range(n_conv):
 layer_in = Conv2D(n_filters, (3,3), padding='same', activation='relu')(layer_in)
 layer_in = MaxPooling2D((2,2), strides=(2,2))(layer_in)
 return layer_in

You can use VGG blocks in any of your models because they are so simple and effective. In the below example the first two blocks have two convolutional layers with 64 and 128 filters respectively, the third block has four convolutional layers with 256 filters. This is a common usage of VGG blocks where the number of filters is increased with the depth of the model.

1
2
3
4
5
6


# We are using functional API just like the previous YOLO implementation
visible = Input(shape=(256, 256, 3))
layer = vgg_block(visible, 64, 2)
layer = vgg_block(layer, 128, 2)
layer = vgg_block(layer, 256, 4)
model = Model(inputs=visible, outputs=layer)

The summary of the model is like this:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31


_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) (None, 256, 256, 3) 0
_________________________________________________________________
conv2d_1 (Conv2D) (None, 256, 256, 64) 1792
_________________________________________________________________
conv2d_2 (Conv2D) (None, 256, 256, 64) 36928
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 128, 128, 64) 0
_________________________________________________________________
conv2d_3 (Conv2D) (None, 128, 128, 128) 73856
_________________________________________________________________
conv2d_4 (Conv2D) (None, 128, 128, 128) 147584
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 64, 64, 128) 0
_________________________________________________________________
conv2d_5 (Conv2D) (None, 64, 64, 256) 295168
_________________________________________________________________
conv2d_6 (Conv2D) (None, 64, 64, 256) 590080
_________________________________________________________________
conv2d_7 (Conv2D) (None, 64, 64, 256) 590080
_________________________________________________________________
conv2d_8 (Conv2D) (None, 64, 64, 256) 590080
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 32, 32, 256) 0
=================================================================
Total params: 2,325,568
Trainable params: 2,325,568
Non-trainable params: 0
_________________________________________________________________

After building the model, all the other works are pretty much the same. You compile the model with an optimizer, fix learning_rate, batch_size, epochs and other hyperparameters. Then fit the model on a dataset.

Inception Model

The inception model was first introduced to the GoogLeNet model back in the 2015 paper. You will be amazed it is still one of the most powerful neural networks out there. Like the VGG model, the inception model also has an inception module which is a block of parallel convolutional layers with different sized filters and a and 3×3 max-pooling layer. The results of all the layers are then concatenated into one layer.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


def inception_module(layer_in, f1, f2, f3):
 # Kernel size increasing at each layer
 conv1 = Conv2D(f1, (1,1), padding='same', activation='relu')(layer_in)
 conv3 = Conv2D(f2, (3,3), padding='same', activation='relu')(layer_in)
 conv5 = Conv2D(f3, (5,5), padding='same', activation='relu')(layer_in)

 pool = MaxPooling2D((3,3), strides=(1,1), padding='same')(layer_in)

 # concatenate filters, assumes filters/channels last
 layer_out = concatenate([conv1, conv3, conv5, pool], axis=-1)
 return layer_out

The function takes the previous layer as input and the kernels for the 3 convolutional layers in an inception module. The concatenated form of these 3 layers is then returned to you for further use.

The rest are the same as VGG network. You need to create a model with multiple linear inception modules in it. As you can see, the model building and summary are just boilerplate codes. The interesting part is the network module layer. This is true for most of the models we will see today.

Residual Model

Residual Model or ResNET is also developed by Google which uses loopback residual blocks. A residual block contains two convolutional layers with the same number of kernels and a small filter size where the output of the second layer is added with the input to the first layer.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


def residual_module(layer_in, n_filters):
 # conv1
 conv1 = Conv2D(n_filters, (3,3), padding='same', activation='relu', kernel_initializer='he_normal')(layer_in)
 # conv2
 conv2 = Conv2D(n_filters, (3,3), padding='same', activation='linear', kernel_initializer='he_normal')(conv1)
 # add filters, assumes filters/channels last
 layer_out = add([conv2, layer_in])
 # activation function
 layer_out = Activation('relu')(layer_out)
 return layer_out

Keep in mind that you need to be careful about the number of filters in the input and output layer. If they do not match, you will most like get an error. The residual network uses one kind of skip connection, which is the unique part of this architecture. In many cases, a lot of primitive features in the network are lost if the model is too deep. Residual network solves this problem with skip connection. The later layers of the model can select them if it needs the output of the layer that is closer to the image, or the previous layer.

Generative Adversarial networks

Generative Adversarial Networks (GANs) are one of the most interesting ideas in computer science today. Two models are trained simultaneously by an adversarial process. A generator (“the artist”) learns to create images that look real, while a discriminator (“the art critic”) learns to tell real images apart from fakes.

The main equation upon which the model is built on is given below:

$$min_G max_D (V_{GAN}(D,G) = E_{x ~ P_{data(x)}} [logD(x)] + E_{z~P_z(z)}[log(1-D(G(z)))]$$

To keep it simple, these are the points you need to keep in mind:

Generator takes a random seed and generates a sample
Discriminator takes an image and tries to detect if that image is from a specific distribution (from the actual dataset).
Discriminator is trained on 2 types of images:
1. Generator generated image with label 0.
2. An image from the dataset with label 1.

GAN is famous for creating AMAZING original photos out of an existing dataset. Below is an image is taken from NVidia, where you can draw anything and it will be transformed into a photorealistic photo of nature.

TensorFlow’s official website has an intuitive guide of how to implement it on TensorFlow. We will follow that guide, but make the code even simpler so you have no problem understanding it. You already understand that we need to design two models for GAN. First, let’s import everything and take the MNIST dataset as an example. GAN is an unsupervised learning model, so we do not need any labels for our data.

Import what we need:

1
2
3
4
5


import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow.keras import layers
import time

And load the data:

1

(train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()

We will create a model where the discriminator will take each image of shape (28,28,1). So the generator has to predict a shape of the same. It can take any size of seed as input, but we will use a linear data of 100 random numbers. Lets reshape and preprocess (simple standardization) the data:

1
2
3
4


# create a grayscale channel which is required for input
train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')
# Normalize the images to [-1, 1]. Previous range was [0,255]
train_images = (train_images - 127.5) / 127.5

Now define some hyperparameters. We will use these variables throughout the code. The names are all self-explanatory

1
2
3


EPOCHS = 50
noise_dim = 100 # seed for generator
BATCH_SIZE = 256

Convert the dataset into a tensorflow dataset:

1

train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(60000).batch(BATCH_SIZE)

To create the generator model we will start with the noise of shape (100,). After a Dense layer, we reshape the model to a 3D shape to match with the image. The height and width will be 7, and the number of channels will be 256. Later we use a layer named transposed convolution, which is basically the opposite of convolution. The rest are almost the same as any other trivial model.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41


def make_generator_model():
 model = tf.keras.Sequential()
 model.add(layers.Dense(7*7*256, use_bias=False, input_shape=(noise_dim,)))
 model.add(layers.BatchNormalization())
 model.add(layers.LeakyReLU())

 model.add(layers.Reshape((7, 7, 256)))
 assert model.output_shape == (None, 7, 7, 256) # None is the batch size

 model.add(layers.Conv2DTranspose(128, (5, 5), strides=(1, 1), padding='same', use_bias=False))
 assert model.output_shape == (None, 7, 7, 128)
 model.add(layers.BatchNormalization())
 model.add(layers.LeakyReLU())

 model.add(layers.Conv2DTranspose(64, (5, 5), strides=(2, 2), padding='same', use_bias=False))
 assert model.output_shape == (None, 14, 14, 64)
 model.add(layers.BatchNormalization())
 model.add(layers.LeakyReLU())

 model.add(layers.Conv2DTranspose(1, (5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))

 # The output must match the dataset images
 assert model.output_shape == (None, 28, 28, 1)

 return model

def make_discriminator_model():
 model = tf.keras.Sequential()
 model.add(layers.Conv2D(64, (5, 5), strides=(2, 2), padding='same',
 input_shape=[28, 28, 1]))
 model.add(layers.LeakyReLU())
 model.add(layers.Dropout(0.3))

 model.add(layers.Conv2D(128, (5, 5), strides=(2, 2), padding='same'))
 model.add(layers.LeakyReLU())
 model.add(layers.Dropout(0.3))

 model.add(layers.Flatten())
 model.add(layers.Dense(1))

 return model

The loss function is the first interesting thing to look at in a GAN. The generator will always be told that its fake inputs are 0 so that it tries to generate better results. The loss of the generator will depend on the output of the discriminator. There will be a lower loss if the discriminator predicts a value close to 1, and 0 otherwise.

For the discriminator, the loss is more straightforward. If the image is from an actual dataset, then the loss is 1. And if the images are coming from the generator, the loss is 0.

Finally, we will use the Adam optimizer with a very small learning rate in the example.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13


cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)

def generator_loss(fake_output):
 return cross_entropy(tf.ones_like(fake_output), fake_output)

def discriminator_loss(real_output, fake_output):
 real_loss = cross_entropy(tf.ones_like(real_output), real_output)
 fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)
 total_loss = real_loss + fake_loss
 return total_loss

generator_optimizer = tf.keras.optimizers.Adam(1e-4)
discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)

The second interesting part of GAN is its training loop. As there are two networks dependent on each other and will be trained simultaneously, the fit method will not work here. We will need to create a custom training loop. While feed forwarding, we keep track of the gradients of each parameter in a GradientTape (separate for each model). After feed forwarding both models, we apply the gradient of loss to both models’ weights.

Note that the training function will need to be compiled so that TensorFlow can build the computation graph first. This helps to speed up the training process and run the function on different devices.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19


# This annotation tells tensorflow to compile the function for training in a device (GPU)
@tf.function
def train_step(images):
 noise = tf.random.normal([BATCH_SIZE, noise_dim])

 with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
 generated_images = generator(noise, training=True)

 real_output = discriminator(images, training=True)
 fake_output = discriminator(generated_images, training=True)

 gen_loss = generator_loss(fake_output)
 disc_loss = discriminator_loss(real_output, fake_output)

 gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)
 gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)

 generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))
 discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))

Now that everything is set, we can run several epochs and provide a batch of images to the train_step function.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


generator = make_generator_model()
discriminator = make_discriminator_model()

def train(epochs):
 for i in range(epochs):
 print('EPOCH: ', i+1)
 for image_batch in train_dataset:
 train_step(image_batch)

train(EPOCHS)

Note: This will take hours to train on this simple dataset. This is the downside of GAN for the moment. It takes a tremendous amount of computational resources and time to train two models simultaneously.

After the training, you can see a sample image generated by creating a seed and passing it through the generator model. You can create as many images as you want (passing the batch size to the model).

1
2


img = generator(tf.random.normal([1, noise_dim]))
plt.imshow(img)

Now you have an artist model which can draw images like a dataset you provided. Cool, right?

Most advanced Machine Learning model (2021): GPT3

GPT-3 is the newest advanced neural network model developed by (OpenAI)[https://openai.com/] which can generate a whole new level of realistic images, videos, code that works by your logic, websites, story, and many more. It is a very deep neural network with 175 billion training parameters. The model is trained in such a way that it can understand the placement of words in any context. It can have a sense of tense, parts of speech, tokens, and tags in programming/scripting languages and many more.

Although GPT-3 is not that useful right now for programmers because is not fully open-sourced yet. But using the API of GPT-3, you can do a lot of cool stuff like telling the program to create an application and it will write the code for your application. So let us see an example of the usage of GPT-3.

The GPT-3 has three variants of language models that we can use.

Zero-shot model: You provide no example and the model will try to figure out what the result should be.
One-shot model: You provide only one example and the model should figure out the result of the next inputs.
Few-shot model: You provide some examples and the model will figure out what the next results should be.

We will use the Few-shot model and the GPT-3 API provided by OpenAI to create a chatbot.

Note: You will need access to the GPT-3 beta program to use the API. Go ahead and join the beta program and request access here.

First, we create the environment and install the only requirement for our project:

1

pip install openai

Start the code by importing and creating a Completion instance of the OpenAI language model. This instance is designed to assist you with different kinds of text completion.

1
2
3
4
5


import os
import openai

openai.api_key = "<your-openai-apikey>"
completion = openai.Completion()

As we will follow the few-shot model, we need an example from which we can start the chat. We will pick up a generalized starting point of the chat. You can change this to something else and see what predictions you get. Just try to select a seed point that is long enough for the model to understand what kind of conversation you want. We will also keep a log of all the messages to provide it to GPT-3 so it can get the context of each new message and try to relate the different pronouns we use.

class chatbot():
def __init__(self):
self.first_msg = '''
Human: Hi there, how are you?
AI: I am great. How can I help you today?
Human:
'''
self.log = self.first_msg
def ask(self, question):
pass

Now we need to send the whole log message to concatenate by the new question of human and send that to GPT-3 via the API. The completion object will help us to create a request and send it over the internet. There are a lot of parameters for its create method. Let us see some of them:

prompt: The input text which needs to be completed
engine: OpenAI has made four text completion engines available, named davinci, ada, babbage and curie. We are using davinci, which is the most capable of the four.
temperature: a number between 0 and 1 that determines how many creative risks the engine takes when generating text.
top_p: an alternative way to control the originality and creativity of the generated text.
frequency_penalty: a number between 0 and 1. The higher this value the model will make a bigger effort in not repeating itself.
presence_penalty: a number between 0 and 1. The higher this value the model will make a bigger effort in talking about new topics.
max_tokens: The maximum completion length.

1
2
3
4
5


 def ask(self):
 response = completion.create(
 prompt=prompt, engine="davinci", stop=['\nHuman'], temperature=0.9,
 top_p=1, frequency_penalty=0, presence_penalty=0.6, best_of=1,
 max_tokens=150)

After sending the response, we need to select the first answer the model gives us. We can return the answer, but before that, we need to update out text log. The final ask function is given below:

1
2
3
4
5
6
7
8
9


 def ask(self, question):
 self.log += question + "\nAI: "
 response = completion.create(
 prompt=self.log, engine="davinci", stop=['\nHuman'], temperature=0.9,
 top_p=1, frequency_penalty=0, presence_penalty=0.6, best_of=1,
 max_tokens=150)
 answer = response.choices[0].text.strip()
 self.log += answer + "\nHuman: "
 return answer

To use this class, we can instantiate and ask some questions.

1
2
3
4
5


cb = chatbot()
cb.ask('Where is Bangladesh Located?')
> 'Bangladesh is a country in Southern Asia and is located on the Bay of Bengal bordered by India'
cb.ask('How long does it take to travel from Los Angeles to Dublin?')
> 'It takes about 12 hours to fly from Los Angeles to Dublin. You may want to fly through Heathrow Airport in London.'

It is awesome, right? Let us know in the discussion section what your conversation with GPT-3 was about.

Conclusion

This lesson was all about making you as much comfortable as possible with a world full of different machine learning models. I hope you will have no problem grabbing a model from the internet, read out its architecture and how it works, and apply it to your own applications.

Machine Learning Tutorial - Lesson 12

rahatzamancse@gmail.com (Rahat Zaman) — Thu, 12 Dec 2019 00:00:00 +0000

Practical Exercise: Create background removing application for Zoom/Skype

Introduction

This lesson is an exercise for everyone to apply machine learning to regular applications that we might need. We will create an application that can take the input from the webcam, remove the background from persons (without any green screen), and then transfer it to a virtual device that can be selected by other applications like Zoom or Skype. We will use only Linux operating system for virtual device creation and provide instructions on how to create DirectShow virtual camera devices on windows using OBS-virtual-Camera DLLs.

General Architecture

The main architecture of the software is given below:

The complete file structure is provided below so that you understand which file to create where:

1
2
3
4
5
6
7


.
├── BackgroundRemove.py : The filter we will use. This is the background remover, you can create anything else you want
├── deeplabv3_mnv2_pascal_train_aug : The large saved model will be here
│   └── saved_model.pb
├── main.py : The entry point
├── v4l2.py : Virtual Device image sending utility
└── VirtualCam.py : Virtual Cam Manager

Virtual Camera

First, we will create a class names VirtualCam.py which will be responsible for all the virtual camera management. It will have a function named send that we will use to send images to the newly created sink device. First, we need to download a utility script for Linux to manage a /dev/video device. Create a project directory with any name you want. Then download the v4l2.py file from here and place it into the root directory of the project.

Now create another file named VirtualCam.py and write a class template inside that file:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21


# We will need these libraries to work in this class
import time
import numpy as np
import os
import fcntl
from v4l2 import (
 v4l2_format, VIDIOC_G_FMT, V4L2_BUF_TYPE_VIDEO_OUTPUT, V4L2_PIX_FMT_RGB24,
 V4L2_FIELD_NONE, VIDIOC_S_FMT
)

class VirtualCam:
 def __init__(self, width, height):
 pass

 # This is needed to clean up the camera when
 # object is destroyed
 def __del__(self):
 pass

 def send(self, frame):
 pass

For now, we will create a virtual camera for Linux-based Operating Systems. At the end of the lesson, I will leave some notes if you want to create a virtual camera for Windows. Because configuring the virtual device on Linux is just simpler than the other, I chose this one to implement so we can focus more on the machine learning/image filtering part.

We need to implement all the functions. First, we will initialize a virtual camera and set the correct properties of the video format we will pass to send to the virtual device. This will allow other applications to understand the camera properties and connect to the camera. A loopback device module will be created and registered to the Linux Kernel. We need a program named v4l2loopback-dkms for this. Install the program using this command:

1

sudo apt-get install -y v4l2loopback-dkms

If you are using an Arch Linux-based distro (e.g. Manjaro), you can install the v4l2loopback-dkms package from AUR. There are a lot of instructions to install this package for other Linux distributions. After installing this, we will have a command named modprobe v4l2loopback which will help us to register a new virtual camera device to the kernel.

1

modprobe v4l2loopback devices=1 exclusive_caps=1 video_nr=5 card_label="vCam"

Note: You will need root privileges to do this. Either you have to always run the python program as root, or you can add pkexec before the command so it will as for admin password when the command runs.

The description of each parameter is given below:

1
2
3
4


device: The number of devices needed for camera
exclusive_caps: Allow other devices to see this camera as external
video_nr: The video number of the camera. This will create a device in /dev/video5 (gives error if it is already occupied)
card_label: The name of the camera, you can change it to whatever you like

After creating the camera, we set its properties (height, width, format, etc.). And in the __del__ method, we delete the device and remove the v4l2loopback kernel module.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17


 def __init__(self, width, height):
 os.system('pkexec modprobe v4l2loopback devices=1 exclusive_caps=1 video_nr=5 card_label="vCam"')
 self.dev = os.open('/dev/video5', os.O_RDWR)

 vid_format = v4l2_format()
 vid_format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT
 fcntl.ioctl(self.dev, VIDIOC_G_FMT, vid_format)
 vid_format.fmt.pix.width = width
 vid_format.fmt.pix.height = height
 vid_format.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB24
 vid_format.fmt.pix.sizeimage = width * height * 3
 vid_format.fmt.pix.field = V4L2_FIELD_NONE
 fcntl.ioctl(self.dev, VIDIOC_S_FMT, vid_format)

 def __del__(self):
 os.close(self.dev)
 os.system('pkexec modprobe -r v4l2loopback')

Finally, we implement the convenient send button that will send an image to the device:

1
2


 def send(self, frame):
 os.write(self.dev, frame.data)

Background Remove

A new filter class will be created where we can send an image and it will apply the desired filter on the image and return it. For that, create a file named BackgroundRemove.py and add this template code to it:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12


import cv2
import numpy as np
import tensorflow as tf

class BackgroundRemover():
 def __init__(self):
 pass


 def apply(self, in_frame):
 # Do the processing
 return in_frame

We will use the Deeplab v3 segmentation model to segment humans from the background. First, download the saved model graph file from here and extract the tar file. Then rename frozen_inference.pb to saved_model.pb. You can delete the other files in the extracted directory. Keep the extracted directory name to whatever you want, but change the name inside the code I show you below so it can find the saved model file.

We need to load the model file into a tf.Graph() object and use it as a function. Unfortunately, the model has trained on the Pascal dataset a long time ago when there was TensorFlow 1. So we need the tf.compat submodule to load the model using in TensorFlow 1 format and run it in a session (an old way to run a computation graph in TensorFlow 1.x).

So in the __init__ method, we create a tf.Graph(), use it as default, and load the model file into it. Then we create a tf.Session (or tf.compat.v1.Session for TF2) and set the graph as default in it.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


 def __init__(self):
 # Loading model into memory
 self.detection_graph = tf.Graph()
 with self.detection_graph.as_default():
 seg_graph_def = tf.compat.v1.GraphDef()
 with tf.io.gfile.GFile('deeplabv3_mnv2_pascal_train_aug/saved_model.pb', 'rb') as fid:
 serialized_graph = fid.read()
 seg_graph_def.ParseFromString(serialized_graph)
 tf.import_graph_def(seg_graph_def, name='')

 self.sess = tf.compat.v1.Session(graph=self.detection_graph)

The graph we just loaded can do a lot of things besides segmentation. But we will only use the segmentation which is labeled as SemanticPrediction in the graph. We set the index of the output of that label as the first (and only) one. So when we run the session, the output result will return a list with only a single element (the SemanticPrediction). The input to the graph is labeled as ImageTensor. So we set it to in_frame as a list. This is done inside the feed_dict parameter in Session.run() method.

The pixels labeled as a human are labeled as 15 in the segmentation map. We will only keep those pixels of the input image, and turn all other pixels to black. See the code below:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13



 def apply(self, in_frame):
 # The graph only takes the image height of 513. And pads the image accordingly
 in_frame = cv2.resize(in_frame, (513, 384))
 with self.detection_graph.as_default():
 batch_seg_map = self.sess.run(
 'SemanticPredictions:0',
 feed_dict={ 'ImageTensor:0': [in_frame] }
 )[0]

 # mask the background
 in_frame[batch_seg_map != 15] = 0
 return in_frame

We have all the parts in place. Now we implement the driver program, the main method. In this method, we read the image, apply the background removing the filter and send it to the virtual camera. We can also see the processed picture to know if it’s working using cv2.imshow() function.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35


import cv2
from VirtualCam import VirtualCam
from BackgroundRemove import BackgroundRemover


def main():
 cap = cv2.VideoCapture(0)
 vcam = VirtualCam(
 width=640,
 height=480
 )
 filter = BackgroundRemover()

 while True:
 # Read the image
 ret, frame = cap.read()
 # Convert to RGB
 frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
 # Apply the filter
 frame = filter.apply(frame)
 # Send to virtual cam
 vcam.send(frame)
 # Display image on screen (Optional)
 cv2.imshow('frame', cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))

 # Break the loop with 'q'
 if cv2.waitKey(1) & 0xFF == ord('q'):
 break

 # Clean up
 cap.release()
 cv2.destroyAllWindows()

if __name__ == '__main__':
 main()

Everything is good to go. Now run the program and you will see a window displayed your camera, Background Removed.

Note: If you see the model running very slowly, it is because you do not have a GPU, or you did the install TensorFlow correctly to use GPU. Please go to the Deep Learning with Tensorflow Lesson and try installing TensorFlow the proper way with CUDA enabled.

You can also keep the program running, and start the zoom application. In the settings, you can see your original camera blank and a new device named “vCam”.

To use it on windows, you will need to create a DLL that can register a DirectShow virtual Camera to the device manager. There is a python module for this here named pyvirtual cam. You can determine the operating system using the platform module in python and dynamically set the virtual camera based on which operating system your program is running on. Below is an example of virtual camera that works for both Windows and Linux. For windows, you will need pyvirtualcam and OBS virtual cam DLLs.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65


import time
import numpy as np
import cv2
from FormatManager import convert_to
from FPSCounter import FPSCounter
import platform

if platform.system() == "Windows":
 from pyvirtualcam import _native_windows
 class _WindowsCamera:
 def send(self, frame: np.ndarray) -> None:
 super().send(frame)
 _native_windows.send(frame)

elif platform.system() == "Linux":
 import os
 import fcntl
 from v4l2 import (
 v4l2_format, VIDIOC_G_FMT, V4L2_BUF_TYPE_VIDEO_OUTPUT, V4L2_PIX_FMT_RGB24,
 V4L2_FIELD_NONE, VIDIOC_S_FMT
 )



class VirtualCam:
 def __init__(self, width, height):
 self.width = width
 self.height = height

 if platform.system() == "Windows":
 _native_windows.start(self.width, self.height, 30, 0)

 elif platform.system() == "Linux":
 self.device_path = "/dev/video5"
 self.dev = os.open(self.device_path, os.O_RDWR)
 vid_format = v4l2_format()
 vid_format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT
 if fcntl.ioctl(self.dev, VIDIOC_G_FMT, vid_format) < 0:
 raise RuntimeError("unable to get output video format")

 vid_format.fmt.pix.width = self.width
 vid_format.fmt.pix.height = self.height
 vid_format.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB24
 vid_format.fmt.pix.sizeimage = self.width * self.height * 3
 vid_format.fmt.pix.field = V4L2_FIELD_NONE

 if fcntl.ioctl(self.dev, VIDIOC_S_FMT, vid_format) < 0:
 raise RuntimeError("unable to set output video format")

 def release(self):
 if platform.system() == "Windows":
 _native_windows.stop()
 elif platform.system() == "Linux":
 os.close(self.dev)

 def __del__(self):
 self.release()

 def send(self, frame, frame_format):
 if platform.system() == "Windows":
 _native_windows.send(frame)

 elif platform.system() == "Linux":
 if os.write(self.dev, frame.data) < 0:
 raise RuntimeError("could not write to output device")

Conclusion

I just showed you how you can apply a machine learning model to your application. Instead of Background Removal, you can also apply other filters that are based on Machine Learning. Let us know different ideas you come up with by sharing them in the discussion section.