Analytic distance metric for Gaussian mixture models

In many applications, you need to compare two or more data sets with each other to see how much they are similar or different. For instance, you have measured the height of men and women in Japan and the Netherlands and now you like to know how much they are different.

Two commonly used method for measuring distances are the Kullback-Liebler divergence and the Bhattacharyya distance.

KL divergence (Kullback-Liebler divergence) measures the difference between two probability distributions p and q.

$https://latex.codecogs.com/svg.image?\LARGE \bg{black}\begin{equation} \label{Kullback_Liebler<em>divergence}D{KL}(p||q)=\int_{-\infty}^\infty p(x)\log\frac{p(x)}{q(x)} \,\mathrm{d}x\end{equation}$

But it only works if your data is made of a single Gaussian and it is not applicable If your data is made of a mixture of Gaussians.

Sfikas et al [1] have extended the Kullback Liebler divergence for GMM and proposed a distance metric using the values ( $https://latex.codecogs.com/svg.image?\LARGE \mu ,\Sigma,\pi$ ) for each one of the two distributions in the following form:

$https://latex.codecogs.com/svg.image?\LARGE \begin{equation} \label{analytical_Kullback_Lieblerdivergence}C2(p||q)=-\log \large[ \frac{2\sum{i,j}\pi{i}\pi{j}^{\prime} \sqrt{ \frac{|V{ij}|}{e^{k{ij}}|\sum{i}| |\sum{j}^{\prime}|} } }{\sum{i,j}\pi{i}\pi{j} \sqrt{ \frac{|V{ij}|}{e^{k{ij}}|\sum{i}| |\sum{j}|} }+\sum{i,j}\pi{i}^{\prime}\pi{j}^{\prime} \sqrt{ \frac{|V{ij}|}{e^{k{ij}}|\sum{i}^{\prime}| |\sum{j}^{\prime}|} }}\large]\end{equation}$

Where:

$https://latex.codecogs.com/svg.image?\LARGE \begin{equation}\label{Kullback_Liebler_divergence<em>Details1}V{ij}=(\Sigma{i}^{-1} +\Sigma{j}^{-1})^{-1}\end{equation}$

and

$https://latex.codecogs.com/svg.image?\LARGE \begin{equation}\label{Kullback_Liebler_divergenceDetails2}K{ij}=\mu{i}^{T}\Sigma{i}^{-1}(\mu{i}-\mu{j}^{\prime})+\mu{j}^{\prime T}\Sigma{j}^{\prime -1}(\mu{j}^{\prime}-\mu{i})\end{equation}$

Code in matlab:

function distance = GetDistanceBetweenTwoDists(means1, pis1, covs1, means2, pis2, covs2)
    %V11 and K11 
    for i=1:size(means1,1)
      for j=1:size(means1,1)
         cov1= covs1(i,:,:);
         cov2= covs1(j,:,:);
         cov1 = reshape(cov1,size(covs1,2), size(covs1,2));
         cov2 = reshape(cov2,size(covs1,2), size(covs1,2)); 
         V11(i,j) = det (inv(inv(cov1)+inv(cov2)));
         K11(i,j) = means1(i,:)*inv(cov1)*(means1(i,:)-means1(j,:))'+means1(j,:)*inv(cov2)*(means1(j,:)-means1(i,:))';
      end
    end

    %V12 and K12
    for i=1:size(means1,1)
      for j=1:size(means2,1)
         cov1= covs1(i,:,:);
         cov2= covs2(j,:,:);
         cov1 = reshape(cov1,size(covs1,2), size(covs1,2));
         cov2 = reshape(cov2,size(covs1,2), size(covs1,2));
         V12(i,j) = det (inv(inv(cov1)+inv(cov2)));
         K12(i,j) = means1(i,:)*inv(cov1)*(means1(i,:)-means2(j,:))'+means2(j,:)*inv(cov2)*(means2(j,:)-means1(i,:))';
      end
    end
    %end

    %V22 and K22
    for i=1:size(means2,1)
      for j=1:size(means2,1)
         cov1= covs2(i,:,:);
         cov2= covs2(j,:,:);
         cov1 = reshape(cov1,size(covs1,2), size(covs1,2));
         cov2 = reshape(cov2,size(covs1,2), size(covs1,2));
         V22(i,j) = det (inv(inv(cov1)+inv(cov2)));
         K22(i,j) = means2(i,:)*inv(cov1)*(means2(i,:)-means2(j,:))'+means2(j,:)*inv(cov2)*(means2(j,:)-means2(i,:))';
      end
    end
    %end

    %Sum11
    Sum11 = 0;

    for i=1:size(means1,1)
      for j=1:size(means1,1)
         cov1= covs1(i,:,:);
         cov2= covs1(j,:,:);
         cov1 = reshape(cov1,size(covs1,2), size(covs1,2));
         cov2 = reshape(cov2,size(covs1,2), size(covs1,2));
         Sum11 = Sum11 + pis1(i)*pis1(j)*sqrt(V11(i,j)/(exp(K11(i,j))*det(cov1)*det(cov2)));
      end
    end

    %Sum12
    Sum12 = 0;

    for i=1:size(means1,1)
      for j=1:size(means2,1)
         cov1= covs1(i,:,:);
         cov2= covs2(j,:,:);
         cov1 = reshape(cov1,size(covs1,2), size(covs1,2));
         cov2 = reshape(cov2,size(covs1,2), size(covs1,2));
         Sum12 = Sum12 + pis1(i)*pis2(j)*sqrt(V12(i,j)/(exp(K12(i,j))*det(cov1)*det(cov2)));
      end
    end

    %Sum22
    Sum22 = 0;

    for i=1:size(means2,1)
      for j=1:size(means2,1)
         cov1= covs2(i,:,:);
         cov2= covs2(j,:,:);
         cov1 = reshape(cov1,size(covs1,2), size(covs1,2));
         cov2 = reshape(cov2,size(covs1,2), size(covs1,2));
         Sum22 = Sum22 + pis2(i)*pis2(j)*sqrt(V22(i,j)/(exp(K22(i,j))*det(cov1)*det(cov2)));
      end
    end

distance = -log(2*Sum12/(Sum11+Sum22));
%distance = 2*Sum12/(Sum11+Sum22);

function distance = GetDistanceBetweenTwoDists(means1, pis1, covs1, means2, pis2, covs2)

%V11 and K11

for i=1:size(means1,1)

for j=1:size(means1,1)

cov1= covs1(i,:,:);

cov2= covs1(j,:,:);

cov1 = reshape(cov1,size(covs1,2), size(covs1,2));

cov2 = reshape(cov2,size(covs1,2), size(covs1,2));

V11(i,j) = det (inv(inv(cov1)+inv(cov2)));

K11(i,j) = means1(i,:)*inv(cov1)*(means1(i,:)-means1(j,:))'+means1(j,:)*inv(cov2)*(means1(j,:)-means1(i,:))';

end

%V12 and K12

for i=1:size(means1,1)

for j=1:size(means2,1)

cov1= covs1(i,:,:);

cov2= covs2(j,:,:);

cov1 = reshape(cov1,size(covs1,2), size(covs1,2));

cov2 = reshape(cov2,size(covs1,2), size(covs1,2));

V12(i,j) = det (inv(inv(cov1)+inv(cov2)));

K12(i,j) = means1(i,:)*inv(cov1)*(means1(i,:)-means2(j,:))'+means2(j,:)*inv(cov2)*(means2(j,:)-means1(i,:))';

end

%end

%V22 and K22

for i=1:size(means2,1)

for j=1:size(means2,1)

cov1= covs2(i,:,:);

cov2= covs2(j,:,:);

cov1 = reshape(cov1,size(covs1,2), size(covs1,2));

cov2 = reshape(cov2,size(covs1,2), size(covs1,2));

V22(i,j) = det (inv(inv(cov1)+inv(cov2)));

K22(i,j) = means2(i,:)*inv(cov1)*(means2(i,:)-means2(j,:))'+means2(j,:)*inv(cov2)*(means2(j,:)-means2(i,:))';

end

%end

%Sum11

Sum11 = 0;

for i=1:size(means1,1)

for j=1:size(means1,1)

cov1= covs1(i,:,:);

cov2= covs1(j,:,:);

cov1 = reshape(cov1,size(covs1,2), size(covs1,2));

cov2 = reshape(cov2,size(covs1,2), size(covs1,2));

Sum11 = Sum11 + pis1(i)*pis1(j)*sqrt(V11(i,j)/(exp(K11(i,j))*det(cov1)*det(cov2)));

end

%Sum12

Sum12 = 0;

for i=1:size(means1,1)

for j=1:size(means2,1)

cov1= covs1(i,:,:);

cov2= covs2(j,:,:);

cov1 = reshape(cov1,size(covs1,2), size(covs1,2));

cov2 = reshape(cov2,size(covs1,2), size(covs1,2));

Sum12 = Sum12 + pis1(i)*pis2(j)*sqrt(V12(i,j)/(exp(K12(i,j))*det(cov1)*det(cov2)));

end

%Sum22

Sum22 = 0;

for i=1:size(means2,1)

for j=1:size(means2,1)

cov1= covs2(i,:,:);

cov2= covs2(j,:,:);

cov1 = reshape(cov1,size(covs1,2), size(covs1,2));

cov2 = reshape(cov2,size(covs1,2), size(covs1,2));

Sum22 = Sum22 + pis2(i)*pis2(j)*sqrt(V22(i,j)/(exp(K22(i,j))*det(cov1)*det(cov2)));

end

distance = -log(2*Sum12/(Sum11+Sum22));

%distance = 2*Sum12/(Sum11+Sum22);

Update: Here is a very nice interactive vizualiztaion of Kullback-Liebler divergence.

Refs: [1]

0 0 votes

Article Rating

Cookie	Duration	Description
cookielawinfo-checbox-analytics	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Analytics".
cookielawinfo-checbox-functional	11 months	The cookie is set by GDPR cookie consent to record the user consent for the cookies in the category "Functional".
cookielawinfo-checbox-others	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Other.
cookielawinfo-checkbox-necessary	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookies is used to store the user consent for the cookies in the category "Necessary".
cookielawinfo-checkbox-performance	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Performance".
viewed_cookie_policy	11 months	The cookie is set by the GDPR Cookie Consent plugin and is used to store whether or not user has consented to the use of cookies. It does not store any personal data.