
    g1                         d Z ddlZddlZddlZddlmZ 	 ddlZn# e	$ r  e
d           Y nw xY wd ZddZd	 Z G d
 d          Z G d de          ZddZ	 ddZ G d de          Zd Zd ZddZ	 	 ddZdS )zO
This contrib module contains a few routines useful to do clustering variants.
    N)
ThreadPoolz2scipy not accessible, Python k-means will not workc                      d S N )argkwargss     V/var/www/html/mpstechhub/venv/lib/python3.11/site-packages/faiss/contrib/clustering.py	print_nopr
      s    D    T   c                 j   | j         d         }|                    dd          }|rt          nt          } |d| j          d| d|             |d           t	          j        ||f|dd	|}	|	                    |            |	j        g}
 |             |	j        } |d
           t          j	                    }|	
                    |           \  }}t          j        ||          } |dt          j	                    |z
  ddt          |           dt          |                      |                                }~	|s3t          j        |dz             |z  |z  }|dd         |dd         z
  }n|t          j        |          }||z  |d         z  }|ddxx         |dd         z  cc<   t%          |          |k    sJ  |dt          |           dt          |                      d}g }t          j	                    }t'          |          D ]}t)          ||                   } |dt          j	                    |z
  dd| d| d| d	dd           |||         z   }|||         }t          j        ||         |k              sJ t	          j        ||fi |}	| |         }|	                    |           |
                    |	j                   |                    |	j                   ~	|} |dt          j	                    |z
  dd           t          j        |          |
fS )a=  
    perform 2-level clustering on a training set xt
    nc1 and nc2 are the number of clusters at each level, the final number of
    clusters is nc2. Additional arguments are passed to the Kmeans object.

    Rebalance allocates the number of sub-clusters depending on the number of
    first-level assignment.
       verboseFz2-level clustering of z nb 1st level clusters = z total zperform coarse trainingi  )nitermax_points_per_centroidzassigning the training set	minlengthzdone in z.2fz s. Sizes of clusters -Nznb 2nd-level centroids r   [z s] training sub-cluster /z nc2= Tendflushz s)shapegetprintr
   faissKmeanstrainiteration_stats	centroidstimeassignnpbincountminmaxargsortarangecumsumsumrangeintallappendvstack)xtnc1nc2	rebalanceclustering_niterargsdr   logkmr#   
centroids1t0_assign1bcoccall_nc2bc_sumi0c2c1i1subsetxtsubs                             r	   two_level_clusteringrL      s    	Ahhy%((G
)%%	CCUUUCUUPSUUVVVC!"""		3
& $
 
 
 
B
 HHRLLL)*OCEEE JC$%%%	B2JAw	W	,	,	,BCR49;;#RRRs2wwRRRRRSSSA
 	EYsQw#%,QRR&2crc7"23,&*,wss|#7||s""""Cc'llCCS\\CCDDD 
B	B	BCjj  '"+U	b UUUrUUCUUcUUU[]eijjjj"R&[2b5vgfo+,,,,,\!S))D))6

r1222
		",C+49;;#++++,,,9R==/))r   c                    t          j        |           } t          | t           j                  rt	          | j                                                  D ]F}| j                            |          }|                    |           |	                    |          }Gt          | j        |fi | d| _        dS t          | t           j                  sJ | j        t           j        k    sJ t!          t#          j        | j                            }t)          d|           t+          ||| j        fi |\  }}| j                            |           | j                            |           |                     |           dS )zJ
    Applies 2-level clustering to an index_ivf embedded in an index.
    TNz
REBALANCE=)r    downcast_index
isinstanceIndexPreTransformr/   chainsizeatr"   applytrain_ivf_index_with_2levelindex
is_trainedIndexIVFmetric_type	METRIC_L2r0   r'   sqrtnlistr   rL   	quantizeradd)rV   r4   r9   ivtr5   r$   r?   s           r	   rU   rU   _   s]   
  ''E%011 u{''))** 	 	A""BHHRLLL"BB#EK<<t<<<eU^,,,,,////
bgek""
#
#C	,'CEEEELIq	O)$$$	O	"""	KKOOOOOr   c                   8    e Zd ZdZd Zd Zd Zd Zd Zd	dZ	dS )
DatasetAssignWrapper for a matrix that offers a function to assign the vectors
    to centroids. All other implementations offer the same interfacec                 <    t          j        |d          | _        d S Nfloat32dtype)r'   ascontiguousarrayxselfrj   s     r	   __init__zDatasetAssign.__init__   s    %ay999r   c                 &    | j         j        d         S )Nr   rj   r   rl   s    r	   countzDatasetAssign.count       v|Ar   c                 &    | j         j        d         S Nr   ro   rp   s    r	   dimzDatasetAssign.dim   rr   r   c                     | j         |         S r   )rj   rl   indicess     r	   
get_subsetzDatasetAssign.get_subset   s    vgr   c                 8    t          j        | j        |d          S rt   )r    knnrj   rl   r$   s     r	   perform_searchzDatasetAssign.perform_search   s    yA...r   Nc                    |                      |          \  }}|                                }|                                }|j        \  }}t          j        ||fd          }|'t          j                            ||| j                   n=t          j                            |||d d t          j        f         | j        z             |||fS re   )	r}   ravelr   r'   zerosr^   rS   rj   newaxis)rl   r$   weightsDIncr:   sum_per_centroids           r	   	assign_tozDatasetAssign.assign_to   s    ""9--1GGIIGGIIA8RG9===?FII&462222FII&7111bj=+ADF+JKKK!%%%r   r   )
__name__
__module____qualname____doc__rm   rq   ru   ry   r}   r   r   r   r	   rb   rb      s        H H: : :      / / /& & & & & &r   rb   c                        e Zd ZdZddZd ZdS )DatasetAssignGPUz GPU version of the previous Fc                    t                               | |           t          j        |j        d                   }|dk    r.t          j        t          j                    ||          | _        d S t          j        |          | _        d S )Nr   r   )	rb   rm   r    IndexFlatL2r   index_cpu_to_gpuStandardGpuResourcesrV   index_cpu_to_all_gpus)rl   rj   gpu_idr   rV   s        r	   rm   zDatasetAssignGPU.__init__   sx    tQ'''!!'!*--Q;;/*,, DJJJ
 4U;;DJJJr   c                     | j                                          | j                             |           | j                             | j        d          S rt   )rV   resetr^   searchrj   r|   s     r	   r}   zDatasetAssignGPU.perform_search   sD    

y!!!z  +++r   N)F)r   r   r   r   rm   r}   r   r   r	   r   r      s=        ''	< 	< 	< 	<, , , , ,r   r   c                    | j         d         }|j         d         }||dz                      d          }|:t          j        |                     d                              d                    }|d| z  |j        z  z
  }|                    d          }|                                |t          j        |          |z  z            |                                z   }||fS )z assignment function for xq is sparse, xb is dense
    uses a matrix multiplication. The squared norms can be provided if
    available.
    r   N   r   )axis)	r   r.   r'   arraypowerTargminr   r,   )	xqxbxq_normsxb_normsnqnbd2r   r   s	            r	   sparse_assign_to_denser      s    
 
!B	!B!G==##8BHHQKKOOA..//
QVbd]
"B
		q	A


1ry}}r))*X^^-=-==Aa4Kr    @  c           
      &   
  j         d         }j         d         t          j        |d          

                    t          j                   t          j        |t                     dz                      d          
 f	d}|dk    s|dk    s|k    r-t          t          |t          d|                               n4t          |          }	|		                    |t          d|                     
fS )z
    decomposes the sparse_assign_to_dense function into blocks to avoid a
    possible memory blow up. Can be run in multithreaded mode, because scipy's
    sparse-dense matrix multiplication is single-threaded.
    r   rf   rg   Nr   r   c           
        	 | | z            }
| | z            }	| | z            };t          j        |                    d                              d                    }n| | z            }t	          d          D ]b}t          |||z            |||z                      \  }}|dk    r||d d <   ||d d <   C||k     }||         |z   ||<   ||         ||<   cd S )Nr   r   r   )r   r   )r'   r   r   r.   r/   r   )r_   xq_blockIblockDblockxq_norms_blockjDiIimaskr   r   bbsr   qbsr   r   r   r   s            r	   handle_query_blockz9sparse_assign_to_dense_blocks.<locals>.handle_query_block   s+   a!c'k?1q3w;1q3w;XhnnQ&7&7&;&;A&>&>??NN%a!c'k2Nq"c"" 	( 	(A+1q3w;'!!a#g+.	  FB Avvqqq	qqq		F{!$x!|t!$xt	( 	(r   )r   r'   emptyfillinfonesr0   r.   listmapr/   r   )r   r   r   r   r   r   ntr   r   poolr   r   r   s   ``````    @@@r	   sparse_assign_to_dense_blocksr      s3    
!B	!B
9%%%AFF26NNN	3			A!G==##( ( ( ( ( ( ( ( ( ( ( ( (. 
Qww"''R3YYS#U1b#%6%6778888"~~#U1b#%6%6777a4Kr   c                   ,    e Zd ZdZd Zd Zd ZddZdS )DatasetAssignSparserc   c                     |j         t          j        j        k    sJ || _        t          j        |                    d                              d                    | _	        d S )Nr   r   )
	__class__scipysparse
csr_matrixrj   r'   r   r   r.   squared_normsrk   s     r	   rm   zDatasetAssignSparse.__init__   sO    {el55555XaggajjnnQ&7&788r   c                 d    t          j        | j        |                                                   S r   )r'   r   rj   todenserw   s     r	   ry   zDatasetAssignSparse.get_subset  s$    xw//11222r   c                 :    t          | j        || j                  S )N)r   )r   rj   r   r|   s     r	   r}   z"DatasetAssignSparse.perform_search  s%    ,FI(:< < < 	<r   Nc                    |                      |          \  }}|                                }|                                }| j        j        d         }|t	          j        |d          }t          |          }t          j        	                    ||t	          j
        |dz             f||f          }t	          j        || j        z                                            }|||fS )Nr   rf   rg   r   )r   )r}   r   rj   r   r'   r   lenr   r   
csc_matrixr,   r   r   )	rl   r$   r   r   r   nr   mr   s	            r	   r   zDatasetAssignSparse.assign_to	  s    ""9--1GGIIGGIIFLO?gay111G^^L##a1q5))*q' $   8QZ$8$8$:$:;;!%%%r   r   )r   r   r   r   rm   ry   r}   r   r   r   r	   r   r      sa        H H9 9 9
3 3 3< < <& & & & & &r   r   c                     t          j        |d          }t          j        t	          |          | t          j        |                    S )Nint64rg   )r'   ri   r    imbalance_factorr   swig_ptr)kr&   s     r	   r   r     s<    !&888F!#f++q%.2H2HIIIr   c                     | j         t          j        k    rdS dd l}t	          | |j                  rdS t          dt          |                      )NFr   TzUnknown tensor type )r   r'   ndarraytorchrO   TensorNotImplementedErrortype)rj   r   s     r	   check_if_torchr      sU    {bj  uLLL!U\"" t
>T!WW>>
?
??r   c                    |t           j        }|j        \  }}d}t          |          }t          j        | dk              d         }t          |          dk    rdS |r ddl}|                    |d                   }	nt          j        |d                   }	|	dddxx         dz  cc<   |	dddxx         dz  cc<   t          |          dk    r|                     d          dz
  }
d|
|
dk     <   |
|
	                                z  }
|
dk    	                                }t          ||j                  }|                    |||
          }t          |d|         |          D ]F\  }}||         }||	z  ||<   ||	z  ||<   | |         dz  | |<   | |xx         | |         z  cc<   |dz  }G||d         }t          |          dk    |S )z/ reassign centroids when some of them collapse Nr   r   g      P?r   float)rR   p)r'   randomr   r   wherer   r   	ones_likeastyper.   r)   rR   choicezip)hassignr$   rsr   r:   nsplitis_torchempty_centsr   facprobasnnznreplacecjscicjcs                    r	   reassign_centroidsr   )  s   	zY?DAqFi((H(7a<((+K
;1q )ooil++l9Q<((!HHH	HHH1IIIIII k

Q

((1,vz&**,,z  sK,--iiFi33+ixi0#66 	 	FB"AGIbMGIbM!"+*GBKBKKK72;&KKKaKFF!()),) k

Q

, Mr     Fc           
         |                                 |                                }}|rt          nt          }	 |	d||| ||fz             t          j                            |          }
t          d           t          j                    }|
                    || d          }|	                    |          }t          |          }g } |	d           d}g }t          |          D ]}t          j                    } |	ddd	
           |                    |          \  }}} |	ddd	
           |t          j                    |z
  z  }|                                }|r|                                }|                    |           t	          j        ||           }|                    dd                              d          }d||dk    <   |r1ddl}|                    |                              |j                  }||z  }t/          |||
          }|t          j                    |z
  |t1          | |          |d} |	d||d         |d         ||d         |fz             |                    |           |? |	d|           |rddl}|                    ||           t	          j        ||           |r||fS |S )a0  Pure python kmeans implementation. Follows the Faiss C++ version
    quite closely, but takes a DatasetAssign instead of a training data
    matrix. Also redo is not implemented.

    For the torch implementation, the centroids are tensors (possibly on GPU),
    but the indices remain numpy on CPU.
    zAClustering %d points in %dD to %d clusters, %d iterations seed %dz
preproc...F)rR   replacez  doner   	assigningr   Tr   zcompute centroidsr   r   r   rf   N)objr%   time_searchr   r   zM  Iteration %d (%.2f s, search %.2f s): objective=%g imbalance=%.3f nsplit=%dr%   r   r   zstoring centroids in)rq   ru   r   r
   r'   r   RandomStater%   r   ry   r   r/   r   r.   itemr2   r(   reshaper   r   
from_numpytodevicer   r   save)r   datar   seed
checkpointr   return_statsr   r:   r;   r   r>   permr$   r   r#   t_search_totr   r_   t0sr&   r   sumserrr   r   r   r   ss                                r	   kmeansr  Z  s    ::<<qA
)%%	CC 
$()1a'=	> ? ? ? 
		t	$	$B	,	B99QQ9..D%%Ii((HOCMMML
C5\\ 1/ 1/ikkKT......334T6666	c))eegg 	((**C

3+f222oob!$$++I66C1H 	8LLL""3''**4;77C3J	#GY;; Y[[2%' 0F ; ;
 
 	 5ai=!1,-9 	
 	
 	
 	q!!!!C&
333 /

9j1111
I... /))r   )Tr   )NN)NNr   r   Nr   )r   r   NTF)r   numpyr'   r    r%   multiprocessing.poolr   scipy.sparser   ImportErrorr   r
   rL   rU   rb   r   r   r   r   r   r   r   r  r   r   r	   <module>r     s          + + + + + +@ @ @ @	E
>?????@	 	 	D* D* D* D*N  F& & & & & & & &D, , , , ,} , , ,(   $ HL- - - -`& & & & &- & & &DJ J J
@ @ @- - - -b CGR R R R R Rs    ..