
    =
j	                        d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZmZ d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Z G d de	      Zy)    )annotationsN)Path)ZipFile)BaseCommandCommandError)transaction)IstatMunicipality)normalize_municipality_namenormalize_province_codec                X    t        | xs d      j                         j                         S N )strstriplowervalues    A/backend/istat/management/commands/import_istat_municipalities.py
_clean_keyr      s#    u{!!#))++    c                D    | t        | xs d      j                         S dS r   )r   r   r   s    r   _clean_valuer      s$    ','83u{!!#@b@r   c                z    | j                         D ci c]  \  }}t        |      t        |       c}}S c c}}w )N)itemsr   r   )rowkeyr   s      r   _normalized_rowr      s2    CF99;O;ZS%JsO\%00;OOOs   7c                z    dD ]  }	 | j                  |      c S  | j                  dd      S # t        $ r Y 6w xY w)N)z	utf-8-sigcp1252latin1r    replace)errors)decodeUnicodeDecodeError)rawencodings     r   _decoder'      sI    5	::h'' 6
 ::hy:11 " 		s   .	::c              #  L  K   t        |       }|d d }	 t        j                         j                  |d      }t        j                  t        j                  |      |      }|D ]  }|st        |        y # t        j                  $ r t        j
                  }Y gw xY ww)Ni   z;,|	)
delimiters)dialect)
r'   csvSniffersniffErrorexcel
DictReaderioStringIOr   )r%   textsampler*   readerr   s         r   _read_csv_rowsr6   $   s     3<D%4[F++-%%f%A ^^BKK-w?F!#&&  99 ))s-   B$%A; 1B$+B$;#B!B$ B!!B$c           	   #    K   	 ddl m}  |t	        j
                  |       dd      }|j                  }|j                  d      }	 t        |      }|D cg c]  }t        |       }}|D ]H  }	t        ||	      D 
ci c]  \  }
}|
t        |       }}
}t        |j                               sE| J y # t        $ r}t        d      |d }~ww xY w# t        $ r Y y w xY wc c}w c c}}
w w)Nr   )load_workbookz7openpyxl is required to import XLSX municipality files.T)	read_only	data_only)values_only)openpyxlr8   ImportErrorr   r1   BytesIOactive	iter_rowsnextStopIterationr   zipr   anyvalues)r%   r8   excworkbook	worksheetrowsheadersheaderkeysrE   r   r   r   s                r   _read_xlsx_rowsrM   1   s     _* RZZ_MHI40Dt* .55W6JvWD5:=dF:KL:KJCsL'':KLszz|I   _TU[^^_  5Lsi   C/B8 <C/C C/C$(C/>C)C/1C/8	CCCC/	C!C/ C!!C/c              #  $  K   | j                   j                         }|dk(  rt        |       5 }t        d |j	                         D        d       }|t        d|        |j                  |      }|j                         j                  d      rt        |      E d {    nt        |      E d {    d d d        y | j                         }|dk(  rt        |      E d {    y t        |      E d {    y 7 ^7 L# 1 sw Y   y xY w7 )7 w)Nz.zipc              3  ^   K   | ]%  }|j                         j                  d       r| ' yw))z.csv.xlsxN)r   endswith).0names     r   	<genexpr>z_read_rows.<locals>.<genexpr>J   s-       2zz|,,->?  2s   +-z!No CSV or XLSX file found inside rP   )suffixr   r   rA   namelistr   readrQ   rM   r6   
read_bytes)pathrU   archivememberr%   s        r   
_read_rowsr\   E   s     [[ FT]g ' 0 0 2
 F ~"%Ftf#MNN,,v&C||~&&w/*3///)#...   	
//
C"3'''!#&&& 0.   	 	(&s_   +DA0D C<D 1C>2D 6,D"D#D6D7D<D >D  D	DDc                @    |D ]  }| j                  |      }|s|c S  yr   )get)r   rL   r   r   s       r   _firstr_   a   s&    L  r   c                   t        | dddddddd	      }t        | d	d
ddd      }t        | ddddd      }t        | ddd      }|r|sy t        |      j                         }|j                         r|j	                  d      }t        |      dk(  r| d}n/t        |      dk  r!|j                         r|j	                  d      }||t        |      t        |      xs d |xs d ddS )Nz"codice comune formato alfanumericozcodice comune formato numericozcodice comunecodice_comunezcodice istatcodice_istatcodecodicezdenominazione in italianozdenominazione comunedenominazionecomunerS   zsigla automobilisticazsigla provinciasigla_provincia	provinciaprovincezdenominazione regioneregioneregion   000	   T)rc   rS   normalized_nameri   rk   	is_active)r_   r   r   isdigitzfilllenr
   r   )r   rc   rS   ri   rk   normalized_codes         r   _municipality_payload_from_rowru   i   s   ,(
D #D H C0)XFFt$ioo'O )//2
?q ,-S1	_		!o&=&=&?)//2  6t<+H5=.D r   c                      e Zd ZdZd Zd Zy)CommandzBImport official ISTAT Italian municipality data into the database.c                ^    |j                  ddt        d       |j                  ddd       y )	Nz--fileTz Official ISTAT CSV/XLSX/ZIP file)requiredtypehelpz--deactivate-missing
store_truezAMark existing municipalities not present in the file as inactive.)actionr{   )add_argumentr   )selfparsers     r   add_argumentszCommand.add_arguments   s6    Ht#Dfg"T 	 	
r   c                r   t        |d         }|j                         st        d|       g }t               }t	        |      D ]<  }t        |      }|r|d   |v r|j                  |d          |j                  |       > |st        d      t        j                         5  t        j                  j                  |      D ci c]  }|j                  | }	}g }
g }|D ]i  }|	j                  |d         }||
j                  t        di |       4|j                         D ]  \  }}t!        |||        |j                  |       k |
r!t        j                  j#                  |
d       |r$t        j                  j%                  |g dd       d	}|d
   r0t        j                  j'                  |      j)                  d      }d d d        | j*                  j-                  | j.                  j1                  dt3        
       dt3               d d             y c c}w # 1 sw Y   _xY w)NfilezMunicipality file not found: rc   z3No municipality rows could be parsed from the file.)code__ini  )
batch_size)rS   ro   ri   rk   rp   r   deactivate_missingF)rp   zImported ISTAT municipalities: z
 created, z
 updated, z deactivated. )r   existsr   setr\   ru   addappendr   atomicr	   objectsfilterrc   r^   r   setattrbulk_createbulk_updateexcludeupdatestdoutwritestyleSUCCESSrs   )r   argsoptionsrY   payloads
seen_codesr   payloadmunicipalityexisting_by_codecreatesupdatesfieldr   deactivateds                  r   handlezCommand.handle   s.   GFO${{}!>tfEFFU
d#C4S9Ggfo;NN76?+OOG$ $ TUU! %6$=$=$D$Dj$D$Y $YL !!</$Y    GG#/33GFOD'NN#4#?w#?@$+MMOLE5L%7 %4|, $ !))55g$5O!))55R# 6  K+,/77??' @ &5&) 7 "> 	JJ1w<.
3w<.
;-}^	
=  "!s   $#H-H(C4H-(H--H6N)__name__
__module____qualname__r{   r   r   r   r   r   rw   rw      s    OD
5
r   rw   )
__future__r   r+   r1   pathlibr   zipfiler   django.core.management.baser   r   	django.dbr   istat.modelsr	   istat.municipalitiesr
   r   r   r   r   r'   r6   rM   r\   r_   ru   rw   r   r   r   <module>r      s^    " 
 	   A ! * U,AP2
'('80f@
k @
r   