python - sentiment - Construyendo una matriz de co-ocurrencia en los pandas de pitón

tweepy sentiment analysis

Question

Sé cómo hacer esto en R Pero, ¿hay alguna función en los pandas que transforme un marco de datos en una matriz de coexistencia de nxn que contenga los recuentos de dos aspectos simultáneos?

Por ejemplo, una matriz df:

import pandas as pd

df = pd.DataFrame({''TFD'' : [''AA'', ''SL'', ''BB'', ''D0'', ''Dk'', ''FF''],
                    ''Snack'' : [''1'', ''0'', ''1'', ''1'', ''0'', ''0''],
                    ''Trans'' : [''1'', ''1'', ''1'', ''0'', ''0'', ''1''],
                    ''Dop'' : [''1'', ''0'', ''1'', ''0'', ''1'', ''1'']}).set_index(''TFD'')

print df

>>> 
    Dop Snack Trans
TFD                
AA    1     1     1
SL    0     0     1
BB    1     1     1
D0    0     1     0
Dk    1     0     0
FF    1     0     1

[6 rows x 3 columns]

cedería:

    Dop Snack Trans

Dop   0     2     3
Snack 2     0     2
Trans 3     2     0

Dado que la matriz se refleja en la diagonal, supongo que habría una forma de optimizar el código.

Answer 1

Demostración en NumPy:

import numpy as np
np.random.seed(3) # for reproducibility

# Generate data: 5 labels, 10 examples, binary.
label_headers = ''Alice Bob Carol Dave Eve''.split('' '')
label_data = np.random.randint(0,2,(10,5)) # binary here but could be any integer.
print(''labels:/n{0}''.format(label_data))

# Compute cooccurrence matrix 
cooccurrence_matrix = np.dot(label_data.transpose(),label_data)
print(''/ncooccurrence_matrix:/n{0}''.format(cooccurrence_matrix)) 

# Compute cooccurrence matrix in percentage
# FYI: http://stackoverflow.com/questions/19602187/numpy-divide-each-row-by-a-vector-element
#      http://stackoverflow.com/questions/26248654/numpy-return-0-with-divide-by-zero/32106804#32106804
cooccurrence_matrix_diagonal = np.diagonal(cooccurrence_matrix)
with np.errstate(divide=''ignore'', invalid=''ignore''):
    cooccurrence_matrix_percentage = np.nan_to_num(np.true_divide(cooccurrence_matrix, cooccurrence_matrix_diagonal[:, None]))
print(''/ncooccurrence_matrix_percentage:/n{0}''.format(cooccurrence_matrix_percentage))

Salida:

labels:
[[0 0 1 1 0]
 [0 0 1 1 1]
 [0 1 1 1 0]
 [1 1 0 0 0]
 [0 1 1 0 0]
 [0 1 0 0 0]
 [0 1 0 1 1]
 [0 1 0 0 1]
 [1 0 0 1 0]
 [1 0 1 1 1]]

cooccurrence_matrix:
[[3 1 1 2 1]
 [1 6 2 2 2]
 [1 2 5 4 2]
 [2 2 4 6 3]
 [1 2 2 3 4]]

cooccurrence_matrix_percentage:
[[ 1.          0.33333333  0.33333333  0.66666667  0.33333333]
 [ 0.16666667  1.          0.33333333  0.33333333  0.33333333]
 [ 0.2         0.4         1.          0.8         0.4       ]
 [ 0.33333333  0.33333333  0.66666667  1.          0.5       ]
 [ 0.25        0.5         0.5         0.75        1.        ]]

Con un mapa de calor usando matplotlib:

import numpy as np
np.random.seed(3) # for reproducibility

import matplotlib.pyplot as plt


def show_values(pc, fmt="%.2f", **kw):
    ''''''
    Heatmap with text in each cell with matplotlib''s pyplot
    Source: http://stackoverflow.com/a/25074150/395857 
    By HYRY
    ''''''
    from itertools import izip
    pc.update_scalarmappable()
    ax = pc.get_axes()
    for p, color, value in izip(pc.get_paths(), pc.get_facecolors(), pc.get_array()):
        x, y = p.vertices[:-2, :].mean(0)
        if np.all(color[:3] > 0.5):
            color = (0.0, 0.0, 0.0)
        else:
            color = (1.0, 1.0, 1.0)
        ax.text(x, y, fmt % value, ha="center", va="center", color=color, **kw)

def cm2inch(*tupl):
    ''''''
    Specify figure size in centimeter in matplotlib
    Source: http://stackoverflow.com/a/22787457/395857
    By gns-ank
    ''''''
    inch = 2.54
    if type(tupl[0]) == tuple:
        return tuple(i/inch for i in tupl[0])
    else:
        return tuple(i/inch for i in tupl)

def heatmap(AUC, title, xlabel, ylabel, xticklabels, yticklabels):
    ''''''
    Inspired by:
    - http://stackoverflow.com/a/16124677/395857 
    - http://stackoverflow.com/a/25074150/395857
    ''''''

    # Plot it out
    fig, ax = plt.subplots()    
    c = ax.pcolor(AUC, edgecolors=''k'', linestyle= ''dashed'', linewidths=0.2, cmap=''RdBu'', vmin=0.0, vmax=1.0)

    # put the major ticks at the middle of each cell
    ax.set_yticks(np.arange(AUC.shape[0]) + 0.5, minor=False)
    ax.set_xticks(np.arange(AUC.shape[1]) + 0.5, minor=False)

    # set tick labels
    #ax.set_xticklabels(np.arange(1,AUC.shape[1]+1), minor=False)
    ax.set_xticklabels(xticklabels, minor=False)
    ax.set_yticklabels(yticklabels, minor=False)

    # set title and x/y labels
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)      

    # Remove last blank column
    plt.xlim( (0, AUC.shape[1]) )

    # Turn off all the ticks
    ax = plt.gca()    
    for t in ax.xaxis.get_major_ticks():
        t.tick1On = False
        t.tick2On = False
    for t in ax.yaxis.get_major_ticks():
        t.tick1On = False
        t.tick2On = False

    # Add color bar
    plt.colorbar(c)

    # Add text in each cell 
    show_values(c)

    # Proper orientation (origin at the top left instead of bottom left)
    ax.invert_yaxis()
    ax.xaxis.tick_top()

    # resize 
    fig = plt.gcf()
    fig.set_size_inches(cm2inch(40, 20))



def main():

    # Generate data: 5 labels, 10 examples, binary.
    label_headers = ''Alice Bob Carol Dave Eve''.split('' '')
    label_data = np.random.randint(0,2,(10,5)) # binary here but could be any integer.
    print(''labels:/n{0}''.format(label_data))

    # Compute cooccurrence matrix 
    cooccurrence_matrix = np.dot(label_data.transpose(),label_data)
    print(''/ncooccurrence_matrix:/n{0}''.format(cooccurrence_matrix)) 

    # Compute cooccurrence matrix in percentage
    # FYI: http://stackoverflow.com/questions/19602187/numpy-divide-each-row-by-a-vector-element
    #      http://stackoverflow.com/questions/26248654/numpy-return-0-with-divide-by-zero/32106804#32106804
    cooccurrence_matrix_diagonal = np.diagonal(cooccurrence_matrix)
    with np.errstate(divide=''ignore'', invalid=''ignore''):
        cooccurrence_matrix_percentage = np.nan_to_num(np.true_divide(cooccurrence_matrix, cooccurrence_matrix_diagonal[:, None]))
    print(''/ncooccurrence_matrix_percentage:/n{0}''.format(cooccurrence_matrix_percentage))

    # Add count in labels
    label_header_with_count = [ ''{0} ({1})''.format(label_header, cooccurrence_matrix_diagonal[label_number]) for label_number, label_header in enumerate(label_headers)]  
    print(''/nlabel_header_with_count: {0}''.format(label_header_with_count))

    # Plotting
    x_axis_size = cooccurrence_matrix_percentage.shape[0]
    y_axis_size = cooccurrence_matrix_percentage.shape[1]
    title = "Co-occurrence matrix/n"
    xlabel= ''''#"Labels"
    ylabel= ''''#"Labels"
    xticklabels = label_header_with_count
    yticklabels = label_header_with_count
    heatmap(cooccurrence_matrix_percentage, title, xlabel, ylabel, xticklabels, yticklabels)
    plt.savefig(''image_output.png'', dpi=300, format=''png'', bbox_inches=''tight'') # use format=''svg'' or ''pdf'' for vectorial pictures
    #plt.show()


if __name__ == "__main__":
    main()
    #cProfile.run(''main()'') # if you want to do some profiling

(PD: una visualización ordenada de una matriz de co-ocurrencia en D3.js )

Answer 2

En caso de que tenga un corpus más grande y una matriz de frecuencia de términos, usar la multiplicación de matriz dispersa podría ser más eficiente. Utilizo el mismo truco de multiplicación de matrices que se refiere a algo en esta página.

import scipy.sparse as sp
X = sp.csr_matrix(df.astype(int).values) # convert dataframe to sparse matrix
Xc = X.T * X # multiply sparse matrix # 
Xc.setdiag(0) # reset diagonal
print(Xc.todense()) # to print co-occurence matrix in dense format

Xc aquí será la matriz de co-ocurrencia en formato csr escaso

Answer 3

Es un álgebra lineal simple, multiplicas la matriz con su transposición (tu ejemplo contiene cadenas, no olvides convertirlas a números enteros):

>>> df_asint = df.astype(int)
>>> coocc = df_asint.T.dot(df_asint)
>>> coocc
       Dop  Snack  Trans
Dop      4      2      3
Snack    2      3      2
Trans    3      2      4

si, como en la respuesta R, quieres restablecer la diagonal, puedes usar fill_diagonal de fill_diagonal :

>>> import numpy as np
>>> np.fill_diagonal(coocc.values, 0)
>>> coocc
       Dop  Snack  Trans
Dop      0      2      3
Snack    2      0      2
Trans    3      2      0