o #j@sPddlZddlZddlZddlmZddlmZgZdZ dZ GdddeZ dS)N)_check_exists_and_download)Datasetz>> import paddle >>> from paddle.text.datasets import Imikolov >>> class SimpleNet(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... ... def forward(self, src, trg): ... return paddle.sum(src), paddle.sum(trg) >>> imikolov = Imikolov(mode='train', data_type='SEQ', window_size=2) >>> for i in range(10): ... src, trg = imikolov[i] ... src = paddle.to_tensor(src) ... trg = paddle.to_tensor(trg) ... ... model = SimpleNet() ... src, trg = model(src, trg) ... print(src.item(), trg.item()) 2076 2075 2076 2075 675 674 4 3 464 463 2076 2075 865 864 2076 2075 2076 2075 1793 1792 NNGRAMtrain2TcCs|dvs Jd|||_|dvsJd|||_||_||_||_|jdurA|s8Jdt|tt d||_| ||_ | dS)N)rSEQz,data type should be 'NGRAM', 'SEQ', but got )rtestz(mode should be 'train', 'test', but got z;data_file is not set and downloading automatically disabledZimikolov) upper data_typelowermode window_size min_word_freq data_filerURLMD5_build_work_dictword_idx _load_anno)selfrr rrrdownloadr^/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/text/datasets/imikolov.py__init__Ts*       zImikolov.__init__cCsb|dur tt}|D]#}|D] }||d7<q|dd7<|dd7<q |S)N) collections defaultdictintstripsplit)rf word_freqlwrrr word_count{s zImikolov.word_countc sd}d}tjQ}||}||}||}d|vr%|d=fdd|D}t|ddd}tt|\} } t tt| t t | } t | | d<Wd| S1s^wY| S) Nz$./simple-examples/data/ptb.train.txtz$./simple-examples/data/ptb.valid.txtcsg|] }|djkr|qS)r)r).0xrrr sz-Imikolov._build_work_dict..cSs|d |dfS)Nrrr)r+rrrsz+Imikolov._build_work_dict..)key) tarfileopenr extractfiler(itemssortedlistzipdictrangelen) rcutoffZtrain_filenameZ test_filenametfZtrainfZtestfr%Zword_freq_sortedwords_rrr,rrs&    zImikolov._build_work_dictc spg_tj}djd}||}jd|D]}jdkrgjdks-Jddg| dg}t |jkrffd d |D}t jt |d D]}j t||j|qTqjd kr| }fd d |D}jdg|}|jdg}jdkrt |jkrqj ||fqtdWddS1swYdS)Nz./simple-examples/data/ptb.z.txtr)rrzInvalid gram lengthrrcg|] }j|qSrrgetr*r'ZUNKrrrr-z'Imikolov._load_anno..rr cr>rr?rArBrrr-rCrzUnknow data type)datar0r1rrr2rr rr"r#r9r8appendtupleAssertionError)rr;filenamer$r&iZsrc_seqZtrg_seqrrBrrs2      "zImikolov._load_annocCstdd|j|DS)NcSsg|]}t|qSr)nparray)r*drrrr-sz(Imikolov.__getitem__..)rFrD)ridxrrr __getitem__szImikolov.__getitem__cCs t|jSN)r9rDr,rrr__len__s zImikolov.__len__)NrrrrTrO) __name__ __module__ __qualname____doc__rr(rrrNrPrrrrrs8 '  r) rr0numpyrJZpaddle.dataset.commonrZ paddle.ior__all__rrrrrrrs