o
    =0fc                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z
dd Zdd Zdd Zdd	 Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! ZdMd#d$Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Z d/d0 Z!d1d2 Z"d3d4 Z#d5d6 Z$d7d8 Z%d9d: Z&d;d< Z'd=d> Z(d?d@ Z)dAdB Z*dCdD Z+dEdF Z,dGdH Z-dIdJ Z.dNdKdLZ/dS )O    Nc                 C   sr   t  }| d | }|dkr|}n| d | | d |  | d |  | d |  }|d}|| | }|S )NV_GTIN V_Name	V_CommentV_Trademarkzutf-8)hashlibmd5encodeupdate	hexdigest)ZdfidxmgtinsnbZskey r   EE:\wamp\www\apps\calme\nutritwin\resources\KCADBProcessorUtilities.py
computeKey   s   0

r   c                 C   sD   | d krd S |  dd}tdd|}tdd|}| dd}|S )Nz&quot;'z[\n\r]*r   u   [®]*\)replaceresub)iStrnr   r   r   removeExoticMarkers   s   r   c           	      C   s   d}z/| d }| d }| d }| d }|dkr+|| | | }t | }| }ntd W n   td td	 td
 t|  td	 Y |d | }|| d< d S )NZXXXX
V_NormNameV_NormCommentV_NormTrademarkr   r   zEmpty name, no possible IDzWrong norm name/commentzK---------------------------------------------------------------------------z	ERROR in:#V_ID)r   r   r	   r   	KCATraces	TRACE_ERRprint)	ZdataRowZ
codeOriginextnamecommentZ	trademarkr   r   txtr   r   r   setRowID&   s*   



r(   c                 C   s@  t | }|d}d}|D ]x}| }|dkrd}ne|dkr!d}n^|dkr(d}nW|d	kr/d
}nP|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dks{|dkr| }|| d }q|d d }|dd  dkr|d d }t|}|S )N r   ZBioZBIOZUhtZUHTzC'EstzC'estZAbcABCZDe   ÀD'ZAuZAuxZLeZLesEZPourL'ZLaZEnZDuZAlZEtZAvec)r   splittitlelowersetCamelCaseFormat)r   r   ZsploStrwr   r   r   
formatNameH   sZ   
8r6   c                 C   sN  | dkrdS d}|  d}g d}|D ]}||v r&|dkr!|d7 }||7 }q|ddks4|ddkr;td|  d}| }|dkrHd	}nL|d
krOd}nE|dkrVd}n>|dkr]d}n7|dkrdd}n0|dkrkd}n)|dkrrd}n"|dkryd}n|dkrd}n|dkrd}n|dkrd}n|dkrd}|dkr|dkr|d7 }||7 }q|S ) Nr   ,)zen:noneen:milken:soybeans en:sulphur-dioxide-and-sulphitesen:nuts
en:peanutszen:lupinz	en:glutenen:eggs
en:mustard	en:celeryzen:molluscsen:crustaceansen:fishen:sesame-seedsen:oaten:r   fr:zWrong allergen: Zlaitr8   Zsojar9   u   fruits à coquesr;   Zoeufr=   ZavoinerC   Z	arachidesr<   Zsulfitesr:   Zmoutarder>   u   célerir?   ZpoissonrA   u
   crustacésr@   u   sésamerB   )r0   findr!   r"   r2   )ZstaZrsrcZaaZalistaZlcar   r   r   formatAllergensh   sX   

rH   c                 C   sX   t | }|}|dkr|dt| 7 }|dd}|dkr*|dt| 7 }|dd}|S )Nr   r)   z   )normalizeNamenormalizeCommentr   normalizeBrand)r%   r&   brandnormNamekeyr   r   r   normalizeConcatenation   s   rO   c                 C   s   | d krdS t | } | S )Nr   normalizeStdr   r   r   r   rJ      s   rJ   c                 C   s4   | d krdS t | } tdd| } tdd| } | S )Nr   \(.*?\)\s\sr)   )rQ   r   r   rR   r   r   r   rI      s   rI   c                 C   s:   | d krdS t | } t| d} | dd} | dd} | S )Nr   Fz sasz s.a.)r   rQ   r   rR   r   r   r   rK      s   
rK   c                 C   sZ   t   t |  t| }|d}d}|D ]}|dkr&|dkr&| }||7 }qt | |S )Nr)   r   )r#   rQ   r0   
capitalize)r   stsoer   r   r   normalizeBrandForDisplay   s   
rZ   c                 C   sd   | d krdS t | }| }g d}|D ]	}t|d|}q| }| }t|}|dd}|S )Nr   )z!\s(sas|sa|plc|p.l.c.|s.a.|s.a.s.)z\s(la\s|)(societe)z\s(des\sproduits)z\s(cooperative)z\s(entreprise)z\s(enseigne)r)   _)rK   r2   r   r   rstriplstripr3   r   )rV   r'   allRegExregr   r   r   normalizeBrandForFileNaming   s   	r`   c                 C   s0   |   } | dd} | dd} | dd} | S )Nr)   -   érY      è)r2   r   rR   r   r   r   normalizeCategories   s
   rd   c                 C   s|   | }g }|  }|d}|D ]}t|}|dkr|| qd}|D ]}|dkr-|d7 }||7 }q#|dd}t|}|S )Nr)   r     )r2   r0   normalizeQuantityWordappendr   reformatQuantity)expZexp2ZallW2ZallWr5   w2r   r   r   normalizeQuantityProposition  s"   


rk   c                 C   s   | }|dkrdS dddddddddddddd}|D ]
}| ||| }qt|d	krS|d
 dkrS|dkrS|dkrS|dkrS|dkrS|dkrS|dkrS|dt|d  }g d}||v r]d}| dd}ddddddddddd
}z|| }W |S    Y |S )Nr   r)   rY   rX   uirG   )ra   zd'une d'un l'u   éerb   rc      ê   ô   û   î   â   à   r/   rV   troisZtiersZquartsZtresplusZgrosr      )zd'uniteZuniteZentierZentierelelaauZeauxZeauundemiZnormalZgarniZpetit)
uneZdemieZgrosseZnormaleZgarnieZpetitededurG   en)r   len)r5   rj   wrerrZwtrZwtwr   r   r   rf   !  s&    H
rf   c                 C   s,  |   }| }dg}d}d}|D ]}|t| }|d7 }|||}qi ddddddd	d
ddddddddddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*
}|D ]
}|||| }qhd}|D ]}|t| }|d7 }|||}qw| }|d+d}|S ),NZquartierzToBeIgnored#ry   rn   r)   d'r    et zun huitiemez1/8ztrois huitiemez3/8zcinq huitiemez5/8zsept huitiemez7/8zun quartz1/4ztrois quartsz3/4Zquartzun tiersz1/3z
deux tiersz2/3zun demiz1/2r~   z0,5z	un moitieZmoitie12345678910)
r}   deuxrw   ZquatreZcinqsixZseptZhuitZneufZdixre   )r2   stripstrr   )rY   r'   ZlistToBeIgnoredsuffixr   Zetir   r   r   r   r   rh   A  sd   rh   c                 C   s   |   } | dd} | dd} tdd| } tdd| } tdd	| } td
d| } tdd| } tdd| } tdd| } | dd} | dd} | S )Nr)   ra   r7      [é|è|ë|ē|ê]rY   
   [ü|ù|û]rl   
   [à|â|ä]rG      [ô|ö]rX      [ç]c   [î|ï]rm      [œ]oez--)r2   r   r   r   rR   r   r   r   normalizeURIi  s   r   Tc                 C   sr  | d krd S d|    d } | dd} | dd} tdd| } tdd| } td	d
| } tdd| } tdd| } tdd| } tdd| } | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} |d krt| } | dkrd|  d } | dd} | S )!Nr)   ra   u   ™r   r   rY   r   rl   r   rG   r   rX   r   r   r   rm   r   r   z & z a z la z le r   z du z au z d'ro   r   "r7   re   T)r2   r   r   r   removePlurial)r   ZisPlurialToBeRemovedr   r   r   rQ   }  s@   rQ   c                 C   s   |  d}ddg}g }|D ]6}t|}||v r|| q|dkrC|d dkr0|d|d  }n|d dkr>|d|d  }|| qd	}|D ]}|d	krR|d7 }||7 }qH|S )
Nr)   r   Znoixry   r/   rV   r   xr   )r0   r   rg   )r   wsZwordExceptionwrr5   Zlwresr   r   r   r     s(   


r   c                 C   s@   t | dkrdS | ddkrdS | d}t |dkrdS dS )Nr   zde:r/   r7   ry   )r   rF   r0   )r   rV   r   r   r   isValidTaxonomy  s   
r   c                 C   s  d}t | }| }|ddksI|ddksI|ddksI|ddksI|ddksI|ddksI|ddksI|d	dksI|d
dkrKdS |ddks|ddks|ddks|ddks|ddks|ddks|ddks|ddks|ddks|ddks|ddks|ddks|ddks|ddks|ddkrdS |ddks|ddkrdS |dd}|d}t|dkrdS |S ) Nry   Z0mlr/   Z0clZ5mlZ5clz1 5z 1lz 2lz1 5 lr   ZbottleZkoffeinfreiz 201u   limitéeZlimitedoriginalZ
direktsaftZschorlezmanzana adesZmoyenzsouth africaZtchequez pckz conz sin ()ra   r)      )rI   r2   rF   r   r0   r   )r   rcrV   ZtabSr   r   r   isValidName  sd   **
r   c                 C   s   g }|  d}|D ]$}|dkr-|ddkr$| }| }|| q	td|  d  q	d}|D ]}|dkr<|d7 }||7 }q2|S )Nr7   r   rE   r/   zInconsistency in the tags: 'r   )r0   rF   r]   r\   rg   r!   r"   )tagsZrTagsZallTagsttagrsr   r   r   
formatTags  s    

r   c                 C   s@   | }t dd|}t dd|}t dd|}t dd|}|S )Nr7   .z[a-z]r   z<\sr)   )r   r   )r   r'   r   r   r   formatValue  s   r   c                 C   s~   |  dd}| }| dd}|d}d}|D ]#}|dkr#|d7 }|ddkr8|ddkr8|d| 7 }q||7 }q|S )	Nr7   /r)   ra   r   rE   r/   rD   )r   r2   r0   rF   )ZcatPathZcatsZallCatsZ
newCatPathr   r   r   r   formatCategories4  s   

r   c                 C   s&  d}d}| dkrd}n| }|  }|dd}|dd}|dd}|d	d
}|dd}t|dksX|d
dkr|ddkr|ddkr|ddkr|ddkr|dd}|dd}|dd}|dd}|dd}|d
}|dkr|d |d  }t|dkrd}nD|d}|dkr|d |d  }t|dkst|dkrd}n$|d}|dkr|d |d  d }|dd}t|dkrd}nd}|dkr|}|d
d}|dd}|dd}|dd}|dd}z	t|}W ||fS    d}d}Y ||fS ||fS )Nr   r   ry   r)   Z1000mlZ1lr7   r   grgZlitrel   r/   mlclZkgZunpotdeZunverreZ	unverredeZ1potdeZunpot      rv   0   
   )r2   r   r   rF   float)r   errr4   Zposi_gZposi_mlZposi_clZstrvfr   r   r   normalizeServingF  sz   




r   c                 C   s   t | }|S NrP   r'   r4   r   r   r   processSportName     r   c                 C      |   }|S r   r2   r   r   r   r   processGenericName  r   r   c                 C   r   r   r   r   r   r   r   normalizeGenericName  r   r   c           
   
   C   s(  t | dkr| S |  }tdd|}tdd|}tdd|}tdd|}td	d
|}tdd|}t|}|dd}g d}d}tdD ]C}|D ]>}z)t||}|d krx| }|	|}t |}	|d| ||t | |	  }W n tj
y   |d7 }Y nw | }qNqJt|}|S )Nr   rS   r   rT   r)   ra   z % z% u   œr   u%   [^a-zA-Z0-9%\s\'àâôîïéèêûü];r7   )z
\s?canettez\s?bouteilleu   \s?sans\sprécisionz*(tablettes?|)\s?[0-9]+(\s|)(x|X)(-|)(\s|$)z\s?-[0-9][0-9]$z\s?[0-9]+gx[0-9]+z(\s?|\+\s?)[0-9]+\s?%$u%   \s?+offre\s(decouverte|sp(é|e)ciale)z\s?+maxi\sformatz\s?\+[0-9]+\s?offerts?$z)\s?(dont|\+|[0-9]+)\s?+[0-9]+\s?offerts?$z\s?prix\schocz\s?[0-9]+cc$z\s?[0-9]+personnes?$z\s?format\sfamilialz'\s?[0-9]+\s?(parts?|sachets?|capsules?)u   \s?[0-9](ème|er)\s(â|a)geuB   \s?(de|)\s?[0-9]+\s?(mois|ans?|)\s?à\s?[0-9]+\s?(mois|ans?)(\s|$)u1   \s?d(ès|e)\s?[0-9]+(/[0-9]|)\s?(mois|ans?)(\s|$)u   \s?(étui|(paquet|colis\sde|sachet|sachet\ssouple)|vrac|boîte|sac|)(\sde\s|\s-\s|\s)([0-9]+?(x|.|)[0-9]+|[0-9]+)\s?(g|gr|grs|l|kg|ml|cl|dz)(\snet|(\s|$))z\s?carton\sde\s[0-9]z9\s?[0-9]+\s?(oz|litre|l|gramme(s|)|gr|kilogramme(s|)|kg)$z\s(bv|a|u|,|-|--)$z\s[x|]\s?[0-9]+x$z\s[x]\s?[0-9]z\.$r   ry   )r   r2   r   r   r   r   rangesearchgrouprF   errorr\   r3   )
r'   ltxtr^   nbRegExErrorrm   rtstooZposiZlgr   r   r   processName  s<   "
 
r   c                 C   s(   t | } | dd} |  }t| } | S Nr   r7   )r   r   r2   r3   )r'   r   r   r   r   processBrand  s
   r   c                 C   s*   | }|  }tdd|}|dd}|S )Nz	[\n\r;_]*r   re   r)   )r2   r   r   r   )rV   r'   r   r   r   processIngredients  s
   r   c                 C   s    t | } |  } | dd} | S r   )r   r2   r   )r'   r   r   r   processComment  s   r   c                 C   sD   d}d}d}|D ]}t || }|r|dd }|}|d7 }q|S )Nr/   r   ry   )r   matchspan)r1   ZregexTabZregPosir   ZlastPosiregexpr   r   r   isRegexCompliant  s   
r   c                 C   s   |  dd}g d}|D ]	}t|d|}q| }| }|d}| }| }|d}|d}| dd}| dd}| dd}| dd}| d	d
}| dd}|S )Nz""z%%)
z((\s|^)cuit(s|es|e|))u   ((\s|^)appertisé(s|es|e|))u   (préemballé(s|es|e|))z((\s|^)rayon\sfrais)u   ((\s|^)sans\sprécision)z((\s|^)aliment\smoyen)u   ((\s|^)à\sréchauffer)z((\s|^)et autres)u   ((\s|^)prêt\sà\sconsommer)z((\s|^)etc)r   r7   r   z,,z, ,z ,z, re   r)   )r   r   r   r]   r\   r   )ZiTxtr'   r^   r_   r   r   r   simplifyComment#  s$   


r   c           
   
   C   s   |  dd dd}dddddddddd		}|D ]
}| ||| }q|d}t|}t|D ]}|| }t|d
krG| }| ||< q0d||< q0d|}	|	S )Nra   r)   r   rY   rX   rl   rm   rG   )	ra   rb   rc   rp   rq   rr   rs   rt   ru   r   r   )r   r0   r   r   r2   rU   join)
r   ZiTextr   r   tabnbwrm   r   ltr'   r   r   r   setSimpleCamelCaseFormatP  s   


r   c                 C   sd  |  d}t|}t|D ]3}|| }t|dkr<| }|dv r*|dkr*|||< q|dv r5| ||< q| ||< qd||< qd}t|D ]}||| 7 }||d kr_|| dkr_|d7 }qG| d}t|}td|D ]}|| d d  || dd   ||< qnd}t|D ]}||| 7 }||d kr|| dkr|d7 }q|dd	}|d
d}|S )Nr)   r   )ru   r   ouZdesetZpourr   r|   Zauxr{   r   Zsansr   )ZaopbioZmgZuhtr   ry   r   r,   r   r.   ro   )r0   r   r   r2   upperrU   r   )r   r   r   rm   r   r   r'   r   r   r   r3   h  s>   



*r3   c              	   C   s  d}| dkr|S |   } t| }| dd dkr"| d| } d|  } | dd} | dd} | d	d} | d
d} | dd} t| }| |d  dksT| |d  dkrZ| d d } | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} |  } |  } g d}d}|D ]*}zt|| }|d krtd| }| }W  |S W q tjy   |d7 }Y qw |S ) Nr   r   r   z1 ry   r   z**u   ?²?u   ?³?   ¹   ²r   *r/   Zgrmr   Zgrsrb   rY   rc   z grammesz grammezpour 1 z	pour une zpour un zpour 2 z
pour deux z g.z gr.)z^[0-9]+$z^\([0-9]+\s?+(g|ml|gr)\)$z^[0-9]+\s?(g|ml|gr)$z^(par|pour|):?\s?(une|deux|trois|)\s?(pot|brique|croque|stick|bouchee|boules?|tranches?|parts?|portions?|cookies?|saucisses?|knackis?|bouchees?|barquettes?|billes?|carres?|madeleines?|barres?|)(\sde\s|\s=\s|\s|)\(?[0-9]+(.|,|)[0-9]\s?(g|ml|gr)\)?$z5^pour [0-9]+\s?(g|ml|gr) de (produit|produit egoutte)z[0-9]+)	r2   r   r   r\   r]   r   r   r   r   )r'   r   Znbcr^   r   r   r   Ztst2r   r   r   processServingSize  sX    	r   c                 C   sX   |dkrt |  t | | D ]}|D ]}|dkr t |d |  ||kr(  dS qqdS )Nr   z    ry   )r#   )ZallNormBrand1ZallNormBrand2Zdbgn1n2r   r   r   isIdenticalBrand  s   r   )T)r   )0stringZcsvsystimeplatformr   jsonior   KCALLMUtilitiesr!   r   r   r(   r6   rH   rO   rJ   rI   rK   rZ   r`   rd   rk   rf   rh   r   rQ   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r3   r   r   r   r   r   r   <module>   sV   (" 6
 (
+!)AZ
	-(?