
    Zg                         d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlZde
e         defdZddZd Zd Zd	 Zd
 ZdS )    N)Listvectorreturnc                 F    t          j        dt          |           z  g| R  S )z=serializes a list of floats into a compact "raw bytes" formatz%sf)structpacklen)r   s    ?/home/debian/html/nutritwin/resources/KCALLMMainGenEmbedding.pyserialize_f32r      s%    ;us6{{*4V4444    c                    i }t          | d          5 }t          j        |          }|                                 d d d            n# 1 swxY w Y   g }d}d}|D ]}|dk    s||         d         s|                    |           t          |          |k    r|dz  }t          dt          |          z              t          j	        |t          j                  }	t          |	          |k    r*t          |          D ]}
|	|
         |||
                  d<   nt          d           t          d           g }t          |          dk    ry|dz  }t          dt          |          z              t          j	        |t          j                  }	t          t          |                    D ]}
|	|
         |||
                  d<   |dk    rt          j        |          }t          j                            |           rt          j        |            t          | d          5 }|                    |           |                                 d d d            n# 1 swxY w Y   t          j                            |          rt          j        |           t)          j        |          }|                    d	           t/          j        |           |                    d
           |                    d                                          \  }}t          d| d|            t          j        }|                    dt          j        |           d           |                                 |                    d           |                                 d}|D ]}|}||         d         }|                    d|t9          |          g           |                    d||g           |dz  }|dz  dk    rt          dt          |          z              |                                 |                                 d S )Nr
   r   s   zRecord #zERROR in dimensionwTFz&select sqlite_version(), vec_version()zsql_version=z, vec_version=z:CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[z])z<CREATE TABLE word_items (key INTEGER PRIMARY KEY, word TEXT)z5INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)z1INSERT INTO word_items(rowid, word) VALUES (?, ?)i  )openjsonloadcloseappendr	   printstrKCALLMEnginegenerateEmbedding
KCALLMCoreMODEL_MISTRAL_ONLINErangeexitdumpsospathexistsremovewritesqlite3connectenable_load_extension
sqlite_vecexecutefetchonegetSignatureSizecommitr   )spathvpathfromScratchsigsfilefoodsnbMax	nbRecordsr   embsisjsonfdicconnsql_versionvec_versionmdlidxtxtsigs                       r
   updateBestSolutionsDBrA      s   D	eS		 Ty

               EEI  !473< LLOOO 5zzU""Q	j3y>>1222#5eZ=\]]t99%%"5\\ 6 6.21gU1Xs++6 .///GGG
5zzQQ	j3y>>)***-eZ5TUUs5zz"" 	* 	*A"&q'DqN3 A~~
4  7>>%   	Ie% 	JJuJJLLL	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 
w~~e 
	%?5!!D 	t$$$ODu%%%  $||,TUU^^``K	
A
A
AK
A
ABBB 
)CLLrjNijmNnNnrrrsssKKMMM 	LLPQQQKKMMM C  1gclLsTabeTfTfNgiiiH3PS*VVVq9>>*s3xx'(((JJLLL
Fs#   )AAA#*II Ic                    | g|z   }t          |          dk    rt          j        |t          j                  }|d         }g }t          dt          |                    D ]=}i }||         |d<   t          |||                   |d<   |                    |           >t          |d           }|D ]#}	t          d|	d          d	|	d                     $d S )
N   r   r   worddistc                     | d         S )NrE    )xs    r
   <lambda>z*searchTheoricalSolutions.<locals>.<lambda>w   s
    qy r   )keyWord: z    -    distance: )
r	   r   r   r   r   r   computeDistancer   sortedr   )
textwordsallWordsr6   embrwr7   r   srwrrs
             r
   searchTheoricalSolutionsrU   g   s    v~H
8}}-h
8WXX1g q#h--(( 	 	AA AfI'T!W55AfIIIaLLLL R00111 	H 	HBF2f:FF"V*FFGGGG
Fr   c                 $   t          j        |           }t          j        |          }t          j        ||          t           j                            |          t           j                            |          z  z  }t          d|z
            }|S )Nr   )nparraydotlinalgnormfloat)t1t2abcosine_similaritycosine_distances         r
   rL   rL   ~   sk    
A
Aq!	q(9(9BINN1<M<M(MNA 1122Or   c           
         t          j        |          }|                    d           t          j        |           |                    d           t          j        | t          j                  }d}t          j
                    }|                    |t          |d                   g          }t          j
                    }||z
  }t          d|dd           |                                }	g }
|	D ]}|\  }}|                    dt          |          z             }|                                }|D ]@}|\  }}t          d	| d
| d|            i }||d<   ||d<   |
                    |           A|                                 |
S )NTFzZSELECT rowid, distance
FROM vec_items
WHERE embedding MATCH ? AND k = 5
ORDER BY distance;r   z--> CPU time in DB: z.4fz secondszf
            SELECT
                rowid, word
            FROM word_items
            WHERE rowid = rK   z  -  dist: z
  -  row: rD   rE   )r&   r'   r(   r)   r   r   r   r   r   timeprocess_timer*   r   r   fetchallr   r   r   )rN   r/   r:   r@   querystartcursorendcpu_timeresultswordTabrowrowidrowdistwordResultswordRowrD   ress                     r
   searchBestSolutionsrt      s    ?5!!D 	t$$$ODu%%%

(z/N
O
OCE
 E\\5M#a&$9$9#:;;F



CU{H	
7
7
7
7
7888ooG G    w !$E

	+
 
 oo''" 	  	 G!KE4F4FFGFFuFFGGGCCK!CKNN3	  	JJLLLNr   c                    t           j                            t          j                    ddd          }t           j                            t          j                    ddd          }|dk    r.t	          d           t	          d           t	          d           nd}|dk    rd	}t          |||           t	          d
           t	          d| z              t	          d
           t	                       t	          d           t          | |          }t	                       t	          d           g }|D ]}|                    |d                    t          | |           d S )Ndata
embeddingszfoodWording.jsonzembeddingDB.sl3r   z,++++++++++++++++++++++++++++++++++++++++++++z#     No forced embedding recreationrC   r   z%-------------------------------------zReference text: z%Results with LOW signature dimension:z&Results with HIGH signature dimension:rD   )	r!   r"   joingetcwdr   rA   rt   r   rU   )rN   
fullUpdater.   r/   
rebuildAllresTabrO   r   s           r
   runUpdateAndRunBasicTestr}      s[    GLLfl<NOOEGLLfl<MNNE Q<===3444<====
??JeUJ777 

1222	
t
#$$$	
1222	GGG	
1222 u--F 
GGG	
2333E    QvYT5)))
Fr   )r   )r   r   r!   r&   r)   numpyrW   r   rd   typingr   r   r\   bytesr   rA   rU   rL   rt   r}   rG   r   r
   <module>r      s         				                     5$u+ 5% 5 5 5 5R R R Rj  .  0 0 0n" " " " "r   