Skip to content

Saliency Map

interpretation.explainer.nn.saliency.SaliencyMap

Bases: NNExplainer

Saliency is a map of the derivatives of the class score with respect to the input pixels / features:

\( \frac{\partial S_c}{\partial x} \)

where \( S_c = \text{class score}\) and \(x = \text{input}\)

It shows the sensitivity of each pixel / feature, higher values can mean higher importance.

Source code in interpretation/explainer/nn/saliency/explainer.py 
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
class SaliencyMap(NNExplainer):
    # make plot work for non 3d image
    r""" 
    Saliency is a map of the derivatives of the class score with respect to the input pixels / features:

    \( \frac{\partial S_c}{\partial x} \)

    where \( S_c = \text{class score}\) and \(x = \text{input}\)

    It shows the sensitivity of each pixel / feature, higher values can mean higher importance.
    """
    def __init__(self, input_model):
        super().__init__(input_model)
        if isinstance(input_model, torch.nn.Module):
            self.istorch = True
        elif isinstance(input_model, tf.keras.Model):
            self.istorch = False
        else:
            raise TypeError(f"`SaliencyMap` only support Torch or TensorFlow models.")

    def compute_map(
        self,
        X: npt.NDArray,
        class_idx: int,
        method: str = "SmoothGrad",
        n_samples: int = 50
    ) -> npt.NDArray:
        """Computes the saliency map of the input

        Parameters
        ----------
        X : npt.NDArray
            Instance to produce the saliency map on
        class_idx : int
            The objective class of the saliency map
        method : str, optional
            Algorithm used for computing the saliency map, by default "SmoothGrad"
        n_samples : int, optional
            Number of samples to average from when using "SmoothGrad", by default 50

        Returns
        -------
        npt.NDArray
            Resulting saliency map, the partial derivative of class score w.r.t. input, shape is the same as the input
        """
        X = validate_input_numpy(X)

        if self.istorch:
            objective_fn = lambda x : x.mean()
        else:
            objective_fn = lambda x : tf.reduce_mean(x)

        if method == "vanilla":
            grad = self.model.compute_gradients(
                X,
                objective=(-1, class_idx),
                objective_fn=objective_fn
            )

        elif method == "SmoothGrad":
            grad = smooth_grad(
                self.model,
                X,
                class_idx=class_idx,
                objective_fn=objective_fn,
                n_samples=n_samples
            )

        else:
            raise ValueError(f"Method {method} is not supported, only valid methods are 'vanilla' and 'SmoothGrad'.")

        return grad

    def plot_map(
        self,
        X: npt.NDArray,
        class_idx: int,
        method: str = "SmoothGrad",
        n_samples: int = 50,
        ax: Axis | None = None
    ):
        """Plots the saliency map of the input on an axis.

        Parameters
        ----------
        X : npt.NDArray
            Instance to produce the saliency map on
        class_idx : int
            The objective class of the saliency map
        method : str, optional
            Algorithm used for computing the saliency map, by default "SmoothGrad"
        n_samples : int, optional
            Number of samples to average from when using "SmoothGrad", by default 50
        ax : Axis | None, optional
            Matplotlib axis to plot on, by default None

        Returns
        -------
        fig : Figure
            Matplotlib Figure object containing the saliency map axis
        ax : Axis
            Matplotlib Axis object that the saliency map is plotted on
        """
        map = self.compute_map(X, class_idx, method, n_samples)

        map = np.transpose(map.squeeze(), (1,2,0))
        map = np.max(np.abs(map), axis=-1)
        map = (map - np.min(map)) / (np.max(map) - np.min(map) + 1e-8)

        if ax is None:
            fig, ax = plt.subplots(figsize=(8, 8))
        else:
            fig = ax.figure

        img = ax.imshow(map, cmap="jet")
        ax.axis("off")

        fig.colorbar(img, ax=ax)

        return fig, ax

compute_map(X, class_idx, method='SmoothGrad', n_samples=50)

Computes the saliency map of the input

Parameters:

Name Type Description Default
X NDArray

Instance to produce the saliency map on

 required
class_idx int

The objective class of the saliency map

 required
method str

Algorithm used for computing the saliency map, by default "SmoothGrad"

 'SmoothGrad'
n_samples int

Number of samples to average from when using "SmoothGrad", by default 50

 50

Returns:

Type Description
NDArray

Resulting saliency map, the partial derivative of class score w.r.t. input, shape is the same as the input
Source code in interpretation/explainer/nn/saliency/explainer.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
def compute_map(
    self,
    X: npt.NDArray,
    class_idx: int,
    method: str = "SmoothGrad",
    n_samples: int = 50
) -> npt.NDArray:
    """Computes the saliency map of the input

    Parameters
    ----------
    X : npt.NDArray
        Instance to produce the saliency map on
    class_idx : int
        The objective class of the saliency map
    method : str, optional
        Algorithm used for computing the saliency map, by default "SmoothGrad"
    n_samples : int, optional
        Number of samples to average from when using "SmoothGrad", by default 50

    Returns
    -------
    npt.NDArray
        Resulting saliency map, the partial derivative of class score w.r.t. input, shape is the same as the input
    """
    X = validate_input_numpy(X)

    if self.istorch:
        objective_fn = lambda x : x.mean()
    else:
        objective_fn = lambda x : tf.reduce_mean(x)

    if method == "vanilla":
        grad = self.model.compute_gradients(
            X,
            objective=(-1, class_idx),
            objective_fn=objective_fn
        )

    elif method == "SmoothGrad":
        grad = smooth_grad(
            self.model,
            X,
            class_idx=class_idx,
            objective_fn=objective_fn,
            n_samples=n_samples
        )

    else:
        raise ValueError(f"Method {method} is not supported, only valid methods are 'vanilla' and 'SmoothGrad'.")

    return grad

plot_map(X, class_idx, method='SmoothGrad', n_samples=50, ax=None)

Plots the saliency map of the input on an axis.

Parameters:

Name Type Description Default
X NDArray

Instance to produce the saliency map on

 required
class_idx int

The objective class of the saliency map

 required
method str

Algorithm used for computing the saliency map, by default "SmoothGrad"

 'SmoothGrad'
n_samples int

Number of samples to average from when using "SmoothGrad", by default 50

 50
ax Axis | None

Matplotlib axis to plot on, by default None

 None

Returns:

Name Type Description
fig Figure

Matplotlib Figure object containing the saliency map axis
ax Axis

Matplotlib Axis object that the saliency map is plotted on
Source code in interpretation/explainer/nn/saliency/explainer.py 
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def plot_map(
    self,
    X: npt.NDArray,
    class_idx: int,
    method: str = "SmoothGrad",
    n_samples: int = 50,
    ax: Axis | None = None
):
    """Plots the saliency map of the input on an axis.

    Parameters
    ----------
    X : npt.NDArray
        Instance to produce the saliency map on
    class_idx : int
        The objective class of the saliency map
    method : str, optional
        Algorithm used for computing the saliency map, by default "SmoothGrad"
    n_samples : int, optional
        Number of samples to average from when using "SmoothGrad", by default 50
    ax : Axis | None, optional
        Matplotlib axis to plot on, by default None

    Returns
    -------
    fig : Figure
        Matplotlib Figure object containing the saliency map axis
    ax : Axis
        Matplotlib Axis object that the saliency map is plotted on
    """
    map = self.compute_map(X, class_idx, method, n_samples)

    map = np.transpose(map.squeeze(), (1,2,0))
    map = np.max(np.abs(map), axis=-1)
    map = (map - np.min(map)) / (np.max(map) - np.min(map) + 1e-8)

    if ax is None:
        fig, ax = plt.subplots(figsize=(8, 8))
    else:
        fig = ax.figure

    img = ax.imshow(map, cmap="jet")
    ax.axis("off")

    fig.colorbar(img, ax=ax)

    return fig, ax