\u8054\u5408\u71b5\u7b49\u4e8e\u7531\u8054\u5408\u4e8b\u4ef6\u53d1\u751f\u6982\u7387\u8ba1\u7b97\u51fa\u7684\u71b5\u3002<\/p>\n\n\n\n
\\[H(X, Y) = E_{X,Y}[-{\\rm log}p(x,y)] = -\\sum_{x,y}p(x,y){\\rm log}p(x,y)\\]<\/p>\n\n\n\n
\u7cfb\u7edf\u6709\u4e2a\u968f\u673a\u53d8\u91cfX\uff0cY\uff0c\u5f53\u7ed9\u5b9a\u6761\u4ef6Y=y\u65f6\uff0cp(X|Y=y)\u7684\u71b5\u4e3a<\/p>\n\n\n\n
\\[{\\rm H}(X|Y=y) = {\\rm H}_{[X|Y]}[-{\\rm log}P(x|y)] = -\\sum_{x\u2208X}p(x|y){\\rm log}p(x|y)\\]<\/p>\n\n\n\n
\u6761\u4ef6\u71b5\u5b9a\u4e49\u4e3a\u4e0a\u9762\u7684\u71b5\u5bf9Y\u5206\u5e03\u7684\u671f\u671b<\/p>\n\n\n\n
\\[{\\rm H}(X) = {\\rm E}_{Y}{\\rm H}[(X|y)] = -\\sum_{y\u2208Y}p(y)\\sum_{x\u2208X}p(x|y){\\rm log}p(x|y) = \\sum_{x,y}p(x,y){\\rm log}\\frac{p(y)}{p(x,y)} \\] <\/p>\n\n\n\n
\u6761\u4ef6\u71b5\u7684\u542b\u4e49\u4e3a\uff1a\u5982\u679c\u77e5\u9053\u4e86Y\uff0cX\u7684\u71b5\u662f\u591a\u5c11\u3002\u6ce8\u610f\u4e0e\u201c\u77e5\u9053Y\u7b49\u4e8e\u4e00\u4e2a\u5177\u4f53\u7684y\u540e\uff0cX\u7684\u71b5\u201d\uff0c\u8fd9\u4e24\u79cd\u8868\u8ff0\u7684\u533a\u5206\u3002\u524d\u8005\u662f\u540e\u8005\u5728Y\u4e0a\u7684\u603b\u4f53\u60c5\u51b5\u3002 <\/p>\n\n\n\n
\u6761\u4ef6\u71b5\u7684\u53e6\u4e00\u79cd\u89e3\u8bfb\u4e3a\uff0c\u77e5\u9053\u4e86Y\u4e4b\u540e\uff0cX\u7684\u4e0d\u786e\u5b9a\u6027\u662f\u591a\u5c11\u3002 \u5c06\u6761\u4ef6\u71b5\u53d8\u5f62\u4e3a\uff1a<\/p>\n\n\n\n
\\[H(X|Y) = H(X, Y) – H(Y)\\]<\/p>\n\n\n\n
\u53ef\u4ee5\u7406\u89e3\u4e3a\uff1a\u201cX\u3001Y\u90fd\u4e0d\u77e5\u9053\u65f6\u7684\u4e0d\u786e\u5b9a\u6027\u201d\u51cf\u53bb\u201cY\u7684\u4e0d\u786e\u5b9a\u6027\u201d\u7b49\u4e8e\u201c\u77e5\u9053Y\u540eX\u7684\u4e0d\u786e\u5b9a\u6027\u201d\u3002<\/p>\n\n\n\n
\u9996\u5148\u4e86\u89e3pointwise mutual information\uff0c\u5b83\u7684\u5b9a\u4e49\u4e3a\uff1a <\/p>\n\n\n\n
\\[{\\rm pmi}(x;y) = {\\rm log}_{2}\\frac{p(x,y)}{p(x)p(y)} = {\\rm log}_{2}\\frac{p(x|y)}{p(x)} = {\\rm log}_{2}\\frac{p(y|x)}{p(y)} \\]<\/p>\n\n\n\n
\u542b\u4e49\u4e3a\u6bd4\u8f83\u4e24\u4e2a\u7ed3\u679c\u540c\u65f6\u53d1\u751f\u7684\u6982\u7387\u4e0e\u4e24\u4e2a\u7ed3\u679c\u5404\u81ea\u53d1\u751f\u6982\u7387\u7684\u4e58\u79ef\uff08\u5373\u5047\u8bbe\u4e24\u4e2a\u4e8b\u4ef6\u72ec\u7acb\uff0c\u5b83\u4eec\u540c\u65f6\u53d1\u751f\u7684\u6982\u7387\uff09\u3002pointwise\u4e92\u4fe1\u606f\u5177\u6709\u5bf9\u79f0\u6027\u3002\u5f53\u5176\u4e3a0\u65f6\u8868\u793a\u4e24\u4e2a\u7ed3\u679c\u662f\u72ec\u7acb\u7684\u3002\u5f53\u5176\u5927\u4e8e0\uff0c\u8868\u793a\u4e24\u4e2a\u7ed3\u679c\u6709\u6b63\u5411\u5173\u8054\uff0c\u5373\u4e00\u4e2a\u53d1\u751f\u4f1a\u5bfc\u81f4\u53e6\u4e00\u4e2a\u66f4\u5bb9\u6613\u53d1\u751f\u3002\u5c0f\u4e8e0\u65f6\u8868\u793a\u6709\u8d1f\u5411\u5173\u8054\uff0c\u4e00\u4e2a\u53d1\u751f\u4f1a\u5bfc\u81f4\u53e6\u4e00\u4e2a\u66f4\u4e0d\u5bb9\u6613\u53d1\u751f\u3002\u5b83\u7684\u4e0a\u4e0b\u754c\u4e3a\uff1a <\/p>\n\n\n\n
\\[-\\infty \u2264 {\\rm pmi}(x;y) \u2264 {\\rm min} [-{\\rm log}p(x), -{\\rm log}p(y)]\\]<\/p>\n\n\n\n
\u4e92\u4fe1\u606f\u662fpointwise \u4e92\u4fe1\u606f\u5728\u6240\u6709\u7ed3\u679c\u4e0a\u7684\u671f\u671b<\/p>\n\n\n\n
\\[I(X;Y) = E_{X,Y}[SI(x,y)] = \\sum_{x,y}p(x,y){\\rm log}\\frac{p(x,y)}{p(x)p(y)}\\]<\/p>\n\n\n\n
\u4ece\u5b9a\u4e49\u7684\u5f62\u5f0f\u4e0a\u53ef\u4ee5\u770b\u51fa\uff0c\u4e92\u4fe1\u606f\u8fd8\u662fP(X, Y)\u548cP(X)*P(Y)\u7684KL\u6563\u5ea6 <\/p>\n\n\n\n
\\[I(X;Y) = KL(P_{(X,Y)}||P(X)\\cdot P(Y))\\]<\/p>\n\n\n\n
\u4e92\u4fe1\u606f\u672c\u8d28\u4e0a\u63cf\u8ff0\u4e86X\u548cY\u7684\u76f8\u5173\u6027\u3002\u8868\u793a\u901a\u8fc7\u77e5\u9053\u4e00\u4e2a\u53d8\u91cf\u7684\u7ed3\u679c\uff0c\u53ef\u4ee5\u51cf\u5c11\u7684\u5173\u4e8e\u53e6\u4e00\u4e2a\u53d8\u91cf\u7684\u4e0d\u786e\u5b9a\u6027\u3002\u5982\u679c\u4e24\u4e2a\u53d8\u91cf\u5b8c\u5168\u72ec\u7acb\uff0c\u90a3\u4e48\u4e92\u4fe1\u606f\u4e3a0\uff0c\u77e5\u9053\u4e00\u4e2a\u53d8\u91cf\u5bf9\u4e8e\u53e6\u4e00\u53d8\u91cf\u7684\u4e0d\u786e\u5b9a\u6027\u5b8c\u5168\u6ca1\u6709\u5e2e\u52a9\u3002\u5982\u679c\u4e24\u4e2a\u53d8\u91cf\u5b8c\u5168\u76f8\u5173\uff0c\u5373\u4e00\u4e2a\u53d8\u91cf\u5b8c\u5168\u51b3\u5b9a\u4e86\u53e6\u4e00\u4e2a\u53d8\u91cf\uff0c\u90a3\u4e48\u4e92\u4fe1\u606f\u5c31\u662f\u5176\u4e2d\u4e00\u4e2a\u53d8\u91cf\u7684\u71b5\u3002 <\/p>\n\n\n\n
\u4e92\u4fe1\u606f\u5177\u6709\u5bf9\u79f0\u6027\uff1b\u4f46\u4e0epoint mutual information\u4e0d\u540c\uff0c\u4e92\u4fe1\u606f\u5177\u6709\u975e\u8d1f\u6027\u3002 <\/p>\n\n\n\n
\u53d8\u5f62\u5f0f<\/p>\n\n\n\n
\\[\\begin{split}
{\\rm I}(X;Y) &= {\\rm H}(X) – {\\rm H}(X|Y)
\\\\ &= {\\rm H}(Y) – {\\rm H}(Y|X)
\\\\ &= {\\rm H}(X) + {\\rm H}(Y) – {\\rm H}(X,Y)
\\\\ &= {\\rm H}(X,Y) – {\\rm H}(X|Y) – {\\rm H}(Y|X)
\\end{split}\\]<\/p>\n\n\n\n
\u4ece\u524d\u4e24\u4e2a\u7b49\u5f0f\u53ef\u4ee5\u76f4\u89c2\u770b\u51fa\uff0c\u4e92\u4fe1\u606f\u8868\u793a\u77e5\u9053\u4e00\u4e2a\u53d8\u91cf\u7684\u7ed3\u679c\uff0c\u51cf\u5c11\u7684\u5173\u4e8e\u53e6\u4e00\u4e2a\u53d8\u91cf\u7684\u4e0d\u786e\u5b9a\u6027\u3002 <\/p>\n","protected":false},"excerpt":{"rendered":"
\u8054\u5408\u71b5(joint entropy) \u8054\u5408\u71b5\u7b49\u4e8e\u7531\u8054\u5408\u4e8b\u4ef6\u53d1\u751f\u6982\u7387\u8ba1\u7b97\u51fa\u7684\u71b5\u3002 \\[H(X, Y) = E_{ … <\/p>\n